From: Peter Amstutz Date: Tue, 25 Aug 2020 18:45:29 +0000 (-0400) Subject: Merge branch '15964-fix-docs' refs #15964 X-Git-Tag: 2.1.0~108 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a7631a1ccb6e2a6925d00a06562e171c4ce4ea2f?hp=bd8bdd90055d61263eff5bdb9a953c57319aa83d Merge branch '15964-fix-docs' refs #15964 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/run-tests.sh b/build/run-tests.sh index bedc95b2db..6c697a657b 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -1012,7 +1012,7 @@ test_doc() { ( set -e cd "$WORKSPACE/doc" - ARVADOS_API_HOST=qr1hi.arvadosapi.com + ARVADOS_API_HOST=pirca.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST diff --git a/doc/_config.yml b/doc/_config.yml index d1c64c61d8..968ca51fa9 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -27,38 +27,37 @@ navbar: - Run a workflow using Workbench: - user/getting_started/workbench.html.textile.liquid - user/tutorials/tutorial-workflow-workbench.html.textile.liquid - - user/composer/composer.html.textile.liquid + - Working at the Command Line: + - user/getting_started/setup-cli.html.textile.liquid + - user/reference/api-tokens.html.textile.liquid + - user/getting_started/check-environment.html.textile.liquid - Access an Arvados virtual machine: - user/getting_started/vm-login-with-webshell.html.textile.liquid - user/getting_started/ssh-access-unix.html.textile.liquid - user/getting_started/ssh-access-windows.html.textile.liquid - - user/getting_started/check-environment.html.textile.liquid - - user/reference/api-tokens.html.textile.liquid - Working with data sets: - user/tutorials/tutorial-keep.html.textile.liquid - user/tutorials/tutorial-keep-get.html.textile.liquid - user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid - user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid - user/tutorials/tutorial-keep-mount-windows.html.textile.liquid - - user/topics/keep.html.textile.liquid - user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid - user/topics/arv-copy.html.textile.liquid - - user/topics/storage-classes.html.textile.liquid - user/topics/collection-versioning.html.textile.liquid - - Working with git repositories: - - user/tutorials/add-new-repository.html.textile.liquid - - user/tutorials/git-arvados-guide.html.textile.liquid - - Running workflows at the command line: + - user/topics/storage-classes.html.textile.liquid + - user/topics/keep.html.textile.liquid + - Data Analysis with Workflows: - user/cwl/cwl-runner.html.textile.liquid - user/cwl/cwl-run-options.html.textile.liquid - - Develop an Arvados workflow: - - user/tutorials/intro-crunch.html.textile.liquid - user/tutorials/writing-cwl-workflow.html.textile.liquid + - user/topics/arv-docker.html.textile.liquid - user/cwl/cwl-style.html.textile.liquid - - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-extensions.html.textile.liquid + - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-versions.html.textile.liquid - - user/topics/arv-docker.html.textile.liquid + - Working with git repositories: + - user/tutorials/add-new-repository.html.textile.liquid + - user/tutorials/git-arvados-guide.html.textile.liquid - Reference: - user/topics/link-accounts.html.textile.liquid - user/reference/cookbook.html.textile.liquid @@ -75,8 +74,9 @@ navbar: - sdk/python/example.html.textile.liquid - sdk/python/python.html.textile.liquid - sdk/python/arvados-fuse.html.textile.liquid - - sdk/python/events.html.textile.liquid + - sdk/python/arvados-cwl-runner.html.textile.liquid - sdk/python/cookbook.html.textile.liquid + - sdk/python/events.html.textile.liquid - CLI: - sdk/cli/install.html.textile.liquid - sdk/cli/index.html.textile.liquid diff --git a/doc/_includes/_0_filter_py.liquid b/doc/_includes/_0_filter_py.liquid deleted file mode 100644 index ff055dbb59..0000000000 --- a/doc/_includes/_0_filter_py.liquid +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -# Import the Arvados sdk module -import arvados - -# Get information about the task from the environment -this_task = arvados.current_task() - -this_task_input = arvados.current_job()['script_parameters']['input'] - -# Create the object access to the collection referred to in the input -collection = arvados.CollectionReader(this_task_input) - -# Create an object to write a new collection as output -out = arvados.CollectionWriter() - -# Create a new file in the output collection -with out.open('0-filter.txt') as out_file: - # Iterate over every input file in the input collection - for input_file in collection.all_files(): - # Output every line in the file that starts with '0' - out_file.writelines(line for line in input_file if line.startswith('0')) - -# Commit the output to Keep. -output_locator = out.finish() - -# Use the resulting locator as the output for this task. -this_task.set_output(output_locator) - -# Done! diff --git a/doc/_includes/_alert-incomplete.liquid b/doc/_includes/_alert-incomplete.liquid deleted file mode 100644 index 8a62ec7a8e..0000000000 --- a/doc/_includes/_alert-incomplete.liquid +++ /dev/null @@ -1,11 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -
- -

Hi!

-

This section is incomplete. Please be patient with us as we fill in the blanks — or contribute to the documentation project.

-
diff --git a/doc/_includes/_alert_stub.liquid b/doc/_includes/_alert_stub.liquid deleted file mode 100644 index dd56f17ddf..0000000000 --- a/doc/_includes/_alert_stub.liquid +++ /dev/null @@ -1,11 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -
- -

Hi!

-

This section is incomplete. Please be patient with us as we fill in the blanks — or contribute to the documentation project.

-
diff --git a/doc/_includes/_arv_copy_expectations.liquid b/doc/_includes/_arv_copy_expectations.liquid deleted file mode 100644 index 2231b06a73..0000000000 --- a/doc/_includes/_arv_copy_expectations.liquid +++ /dev/null @@ -1,12 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{% include 'notebox_begin' %} -As stated above, arv-copy is recursive by default and requires a working git repository in the destination cluster. If you do not have a repository created, you can follow the "Adding a new repository":{{site.baseurl}}/user/tutorials/add-new-repository.html page. We will use the *tutorial* repository created in that page as the example. - -
In addition, arv-copy requires git when copying to a git repository. Please make sure that git is installed and available. - -{% include 'notebox_end' %} diff --git a/doc/_includes/_concurrent_hash_script_py.liquid b/doc/_includes/_concurrent_hash_script_py.liquid deleted file mode 100644 index 2c55298841..0000000000 --- a/doc/_includes/_concurrent_hash_script_py.liquid +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -import hashlib -import os -import arvados - -# Jobs consist of one or more tasks. A task is a single invocation of -# a crunch script. - -# Get the current task -this_task = arvados.current_task() - -# Tasks have a sequence number for ordering. All tasks -# with the current sequence number must finish successfully -# before tasks in the next sequence are started. -# The first task has sequence number 0 -if this_task['sequence'] == 0: - # Get the "input" field from "script_parameters" on the task object - job_input = arvados.current_job()['script_parameters']['input'] - - # Create a collection reader to read the input - cr = arvados.CollectionReader(job_input) - - # Loop over each stream in the collection (a stream is a subset of - # files that logically represents a directory) - for s in cr.all_streams(): - - # Loop over each file in the stream - for f in s.all_files(): - - # Synthesize a manifest for just this file - task_input = f.as_manifest() - - # Set attributes for a new task: - # 'job_uuid' the job that this task is part of - # 'created_by_job_task_uuid' this task that is creating the new task - # 'sequence' the sequence number of the new task - # 'parameters' the parameters to be passed to the new task - new_task_attrs = { - 'job_uuid': arvados.current_job()['uuid'], - 'created_by_job_task_uuid': arvados.current_task()['uuid'], - 'sequence': 1, - 'parameters': { - 'input':task_input - } - } - - # Ask the Arvados API server to create a new task, running the same - # script as the parent task specified in 'created_by_job_task_uuid' - arvados.api().job_tasks().create(body=new_task_attrs).execute() - - # Now tell the Arvados API server that this task executed successfully, - # even though it doesn't have any output. - this_task.set_output(None) -else: - # The task sequence was not 0, so it must be a parallel worker task - # created by the first task - - # Instead of getting "input" from the "script_parameters" field of - # the job object, we get it from the "parameters" field of the - # task object - this_task_input = this_task['parameters']['input'] - - collection = arvados.CollectionReader(this_task_input) - - # There should only be one file in the collection, so get the - # first one from the all files iterator. - input_file = next(collection.all_files()) - output_path = os.path.normpath(os.path.join(input_file.stream_name(), - input_file.name)) - - # Everything after this is the same as the first tutorial. - digestor = hashlib.new('md5') - for buf in input_file.readall(): - digestor.update(buf) - - out = arvados.CollectionWriter() - with out.open('md5sum.txt') as out_file: - out_file.write("{} {}\n".format(digestor.hexdigest(), output_path)) - - this_task.set_output(out.finish()) - -# Done! diff --git a/doc/_includes/_crunch1only_begin.liquid b/doc/_includes/_crunch1only_begin.liquid deleted file mode 100644 index 6dc304a928..0000000000 --- a/doc/_includes/_crunch1only_begin.liquid +++ /dev/null @@ -1,8 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{% include 'notebox_begin_warning' %} -This section assumes the legacy Jobs API is available. Some newer installations have already disabled the Jobs API in favor of the Containers API. diff --git a/doc/_includes/_crunch1only_end.liquid b/doc/_includes/_crunch1only_end.liquid deleted file mode 100644 index a3f2278fd3..0000000000 --- a/doc/_includes/_crunch1only_end.liquid +++ /dev/null @@ -1,7 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{% include 'notebox_end' %} diff --git a/doc/_includes/_example_docker.liquid b/doc/_includes/_example_docker.liquid deleted file mode 100644 index 2d6335a62b..0000000000 --- a/doc/_includes/_example_docker.liquid +++ /dev/null @@ -1,34 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ - "name": "Example using R in a custom Docker image", - "components": { - "Rscript": { - "script": "run-command", - "script_version": "master", - "repository": "arvados", - "script_parameters": { - "command": [ - "Rscript", - "$(glob $(file $(myscript))/*.r)", - "$(glob $(dir $(mydata))/*.csv)" - ], - "myscript": { - "required": true, - "dataclass": "Collection" - }, - "mydata": { - "required": true, - "dataclass": "Collection" - } - }, - "runtime_constraints": { - "docker_image": "arvados/jobs-with-r" - } - } - } -} diff --git a/doc/_includes/_install_compute_docker.liquid b/doc/_includes/_install_compute_docker.liquid index 63c54aed72..fd5d88a9c3 100644 --- a/doc/_includes/_install_compute_docker.liquid +++ b/doc/_includes/_install_compute_docker.liquid @@ -6,7 +6,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0 h2(#cgroups). Configure Linux cgroups accounting -Linux can report what compute resources are used by processes in a specific cgroup or Docker container. Crunch can use these reports to share that information with users running compute work. This can help pipeline authors debug and optimize their workflows. +Linux can report what compute resources are used by processes in a specific cgroup or Docker container. Crunch can use these reports to share that information with users running compute work. This can help workflow authors debug and optimize their workflows. To enable cgroups accounting, you must boot Linux with the command line parameters @cgroup_enable=memory swapaccount=1@. diff --git a/doc/_includes/_install_rails_reconfigure.liquid b/doc/_includes/_install_rails_reconfigure.liquid deleted file mode 100644 index 4687431c2a..0000000000 --- a/doc/_includes/_install_rails_reconfigure.liquid +++ /dev/null @@ -1,17 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -Now that all your configuration is in place, rerun the {{railspkg}} package configuration to install necessary Ruby Gems and other server dependencies. On Debian-based systems: - -
~$ sudo dpkg-reconfigure {{railspkg}}
-
 - -On Red Hat-based systems: - -
~$ sudo yum reinstall {{railspkg}}
-
 - -You only need to do this manual step once, after initial configuration. When you make configuration changes in the future, you just need to restart Nginx for them to take effect. \ No newline at end of file diff --git a/doc/_includes/_install_ruby_and_bundler_sso.liquid b/doc/_includes/_install_ruby_and_bundler_sso.liquid deleted file mode 100644 index a8d14efdee..0000000000 --- a/doc/_includes/_install_ruby_and_bundler_sso.liquid +++ /dev/null @@ -1,69 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -Ruby 2.3 is recommended; Ruby 2.1 is also known to work. - -h4(#rvm). *Option 1: Install with RVM* - - -
sudo gpg --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.3
-
 - -Either log out and log back in to activate RVM, or explicitly load it in all open shells like this: - - -
source /usr/local/rvm/scripts/rvm
-
 - -Once RVM is activated in your shell, install Bundler: - - -
~$ gem install bundler
-
 - -h4(#fromsource). *Option 2: Install from source* - -Install prerequisites for Debian 8: - - -
sudo apt-get install \
-    bison build-essential gettext libcurl3 libcurl3-gnutls \
-    libcurl4-openssl-dev libpcre3-dev libreadline-dev \
-    libssl-dev libxslt1.1 zlib1g-dev
-
 - -Install prerequisites for CentOS 7: - - -
sudo yum install \
-    libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel \
-    patch readline-devel zlib-devel libffi-devel openssl-devel \
-    make automake libtool bison sqlite-devel tar
-
 - -Install prerequisites for Ubuntu 12.04 or 14.04: - - -
sudo apt-get install \
-    gawk g++ gcc make libc6-dev libreadline6-dev zlib1g-dev libssl-dev \
-    libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev \
-    libncurses5-dev automake libtool bison pkg-config libffi-dev curl
-
 - -Build and install Ruby: - - -
mkdir -p ~/src
-cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz | tar xz
-cd ruby-2.3.3
-./configure --disable-install-rdoc
-make
-sudo make install
-
-sudo -i gem install bundler
-
 diff --git a/doc/_includes/_install_runit.liquid b/doc/_includes/_install_runit.liquid deleted file mode 100644 index d5f8341311..0000000000 --- a/doc/_includes/_install_runit.liquid +++ /dev/null @@ -1,19 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -On Debian-based systems: - - -
~$ sudo apt-get install runit
-
- - -On Red Hat-based systems: - - -
~$ sudo yum install runit
-
- diff --git a/doc/_includes/_pipeline_deprecation_notice.liquid b/doc/_includes/_pipeline_deprecation_notice.liquid deleted file mode 100644 index 35c89be447..0000000000 --- a/doc/_includes/_pipeline_deprecation_notice.liquid +++ /dev/null @@ -1,9 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{% include 'notebox_begin_warning' %} -Arvados pipeline templates are deprecated. The recommended way to develop new workflows for Arvados is using the "Common Workflow Language":{{site.baseurl}}/user/cwl/cwl-runner.html. -{% include 'notebox_end' %} diff --git a/doc/_includes/_run_command_foreach_example.liquid b/doc/_includes/_run_command_foreach_example.liquid deleted file mode 100644 index 8e3dd713d1..0000000000 --- a/doc/_includes/_run_command_foreach_example.liquid +++ /dev/null @@ -1,46 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ - "name":"run-command example pipeline", - "components":{ - "bwa-mem": { - "script": "run-command", - "script_version": "master", - "repository": "arvados", - "script_parameters": { - "command": [ - "bwa", - "mem", - "-t", - "$(node.cores)", - "$(glob $(dir $(reference_collection))/*.fasta)", - { - "foreach": "read_pair", - "command": "$(read_pair)" - } - ], - "task.stdout": "$(basename $(glob $(dir $(sample))/*_1.fastq)).sam", - "task.foreach": ["sample_subdir", "read_pair"], - "reference_collection": { - "required": true, - "dataclass": "Collection" - }, - "sample": { - "required": true, - "dataclass": "Collection" - }, - "sample_subdir": "$(dir $(sample))", - "read_pair": { - "value": { - "group": "sample_subdir", - "regex": "(.*)_[12]\\.fastq(\\.gz)?$" - } - } - } - } - } -} diff --git a/doc/_includes/_run_command_simple_example.liquid b/doc/_includes/_run_command_simple_example.liquid deleted file mode 100644 index b37ae9a88b..0000000000 --- a/doc/_includes/_run_command_simple_example.liquid +++ /dev/null @@ -1,43 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ - "name":"run-command example pipeline", - "components":{ - "bwa-mem": { - "script": "run-command", - "script_version": "master", - "repository": "arvados", - "script_parameters": { - "command": [ - "$(dir $(bwa_collection))/bwa", - "mem", - "-t", - "$(node.cores)", - "-R", - "@RG\\\tID:group_id\\\tPL:illumina\\\tSM:sample_id", - "$(glob $(dir $(reference_collection))/*.fasta)", - "$(glob $(dir $(sample))/*_1.fastq)", - "$(glob $(dir $(sample))/*_2.fastq)" - ], - "reference_collection": { - "required": true, - "dataclass": "Collection" - }, - "bwa_collection": { - "required": true, - "dataclass": "Collection", - "default": "39c6f22d40001074f4200a72559ae7eb+5745" - }, - "sample": { - "required": true, - "dataclass": "Collection" - }, - "task.stdout": "$(basename $(glob $(dir $(sample))/*_1.fastq)).sam" - } - } - } -} diff --git a/doc/_includes/_run_md5sum_py.liquid b/doc/_includes/_run_md5sum_py.liquid deleted file mode 100644 index 6d10672db6..0000000000 --- a/doc/_includes/_run_md5sum_py.liquid +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -import arvados - -# Automatically parallelize this job by running one task per file. -arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True, - input_as_path=True) - -# Get the input file for the task -input_file = arvados.get_task_param_mount('input') - -# Run the external 'md5sum' program on the input file -stdoutdata, stderrdata = arvados.util.run_command(['md5sum', input_file]) - -# Save the standard output (stdoutdata) to "md5sum.txt" in the output collection -out = arvados.CollectionWriter() -with out.open('md5sum.txt') as out_file: - out_file.write(stdoutdata) -arvados.current_task().set_output(out.finish()) diff --git a/doc/_includes/_ssh_addkey.liquid b/doc/_includes/_ssh_addkey.liquid index 7a8a992b68..de0da6a767 100644 --- a/doc/_includes/_ssh_addkey.liquid +++ b/doc/_includes/_ssh_addkey.liquid @@ -18,6 +18,10 @@ Paste your public key into the text area labeled *Public Key*, and click on the h1(#login). Using SSH to log into an Arvados VM -To see a list of virtual machines that you have access to and determine the name and login information, click on the dropdown menu icon in the upper right corner of the top navigation menu to access the user settings menu and click on the menu item *Virtual machines* to go to the Virtual machines page. This page lists the virtual machines you can access. The *Host name* column lists the name of each available VM. The *Login name* column will have a list of comma separated values of the form @you@. In this guide the hostname will be *_shell_* and the login will be *_you_*. Replace these with your hostname and login name as appropriate. +To see a list of virtual machines that you have access to, click on the dropdown menu icon in the upper right corner of the top navigation menu to access the user settings menu, then click on the menu item *Virtual machines* to go to the Virtual machines page. +This page lists the virtual machines you can access. The *Host name* column lists the name of each available VM. The *Login name* column lists your login name on that VM. The *Command line* column provides a sample @ssh@ command line. +At the bottom of the page there may be additional instructions for connecting your specific Arvados instance. If so, follow your site-specific instructions. If there are no site-specific instructions, you can probably connect directly with @ssh@. + +The following are generic instructions. In the examples the login will be *_you_* and the hostname will be *_shell.ClusterID.example.com_* and . Replace these with your login name and hostname as appropriate. diff --git a/doc/_includes/_tutorial_bwa_sortsam_pipeline.liquid b/doc/_includes/_tutorial_bwa_sortsam_pipeline.liquid deleted file mode 100644 index 3b39403488..0000000000 --- a/doc/_includes/_tutorial_bwa_sortsam_pipeline.liquid +++ /dev/null @@ -1,78 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ - "name": "Tutorial align using bwa mem and SortSam", - "components": { - "bwa-mem": { - "script": "run-command", - "script_version": "master", - "repository": "arvados", - "script_parameters": { - "command": [ - "$(dir $(bwa_collection))/bwa", - "mem", - "-t", - "$(node.cores)", - "-R", - "@RG\\\tID:group_id\\\tPL:illumina\\\tSM:sample_id", - "$(glob $(dir $(reference_collection))/*.fasta)", - "$(glob $(dir $(sample))/*_1.fastq)", - "$(glob $(dir $(sample))/*_2.fastq)" - ], - "reference_collection": { - "required": true, - "dataclass": "Collection" - }, - "bwa_collection": { - "required": true, - "dataclass": "Collection", - "default": "39c6f22d40001074f4200a72559ae7eb+5745" - }, - "sample": { - "required": true, - "dataclass": "Collection" - }, - "task.stdout": "$(basename $(glob $(dir $(sample))/*_1.fastq)).sam" - }, - "runtime_constraints": { - "docker_image": "bcosc/arv-base-java", - "arvados_sdk_version": "master" - } - }, - "SortSam": { - "script": "run-command", - "script_version": "847459b3c257aba65df3e0cbf6777f7148542af2", - "repository": "arvados", - "script_parameters": { - "command": [ - "java", - "-Xmx4g", - "-Djava.io.tmpdir=$(tmpdir)", - "-jar", - "$(dir $(picard))/SortSam.jar", - "CREATE_INDEX=True", - "SORT_ORDER=coordinate", - "VALIDATION_STRINGENCY=LENIENT", - "INPUT=$(glob $(dir $(input))/*.sam)", - "OUTPUT=$(basename $(glob $(dir $(input))/*.sam)).sort.bam" - ], - "input": { - "output_of": "bwa-mem" - }, - "picard": { - "required": true, - "dataclass": "Collection", - "default": "88447c464574ad7f79e551070043f9a9+1970" - } - }, - "runtime_constraints": { - "docker_image": "bcosc/arv-base-java", - "arvados_sdk_version": "master" - } - } - } -} diff --git a/doc/_includes/_tutorial_cluster_name.liquid b/doc/_includes/_tutorial_cluster_name.liquid deleted file mode 100644 index 22fbc463e6..0000000000 --- a/doc/_includes/_tutorial_cluster_name.liquid +++ /dev/null @@ -1,9 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{% include 'notebox_begin' %} -This tutorial assumes you are using the default Arvados instance, @qr1hi@. If you are using a different instance, replace @qr1hi@ with your instance. See "Accessing Arvados Workbench":{{site.baseurl}}/user/getting_started/workbench.html for more details. -{% include 'notebox_end' %} diff --git a/doc/_includes/_tutorial_expectations.liquid b/doc/_includes/_tutorial_expectations.liquid index 6c4fbeb1f3..09b18f0d4d 100644 --- a/doc/_includes/_tutorial_expectations.liquid +++ b/doc/_includes/_tutorial_expectations.liquid @@ -5,5 +5,5 @@ SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} {% include 'notebox_begin' %} -This tutorial assumes that you are logged into an Arvados VM instance (instructions for "Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html or "Unix":{{site.baseurl}}/user/getting_started/ssh-access-unix.html#login or "Windows":{{site.baseurl}}/user/getting_started/ssh-access-windows.html#login) or you have installed the Arvados "FUSE Driver":{{site.baseurl}}/sdk/python/arvados-fuse.html and "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html on your workstation and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html +This tutorial assumes that you have access to the "Arvados command line tools":/user/getting_started/setup-cli.html and have set the "API token":{{site.baseurl}}/user/reference/api-tokens.html and confirmed a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html . {% include 'notebox_end' %} diff --git a/doc/_includes/_tutorial_hash_script_py.liquid b/doc/_includes/_tutorial_hash_script_py.liquid deleted file mode 100644 index 9eacb763dc..0000000000 --- a/doc/_includes/_tutorial_hash_script_py.liquid +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -import hashlib # Import the hashlib module to compute MD5. -import os # Import the os module for basic path manipulation -import arvados # Import the Arvados sdk module - -# Automatically parallelize this job by running one task per file. -# This means that if the input consists of many files, each file will -# be processed in parallel on different nodes enabling the job to -# be completed quicker. -arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True, - input_as_path=True) - -# Get object representing the current task -this_task = arvados.current_task() - -# Create the message digest object that will compute the MD5 hash -digestor = hashlib.new('md5') - -# Get the input file for the task -input_id, input_path = this_task['parameters']['input'].split('/', 1) - -# Open the input collection -input_collection = arvados.CollectionReader(input_id) - -# Open the input file for reading -with input_collection.open(input_path) as input_file: - for buf in input_file.readall(): # Iterate the file's data blocks - digestor.update(buf) # Update the MD5 hash object - -# Write a new collection as output -out = arvados.CollectionWriter() - -# Write an output file with one line: the MD5 value and input path -with out.open('md5sum.txt') as out_file: - out_file.write("{} {}/{}\n".format(digestor.hexdigest(), input_id, - os.path.normpath(input_path))) - -# Commit the output to Keep. -output_locator = out.finish() - -# Use the resulting locator as the output for this task. -this_task.set_output(output_locator) - -# Done! diff --git a/doc/_includes/_install_git.liquid b/doc/_includes/_tutorial_hello_cwl.liquid similarity index 50% rename from doc/_includes/_install_git.liquid rename to doc/_includes/_tutorial_hello_cwl.liquid index d60379fa30..ae1ec80ab2 100644 --- a/doc/_includes/_install_git.liquid +++ b/doc/_includes/_tutorial_hello_cwl.liquid @@ -1,9 +1,11 @@ +#!/usr/bin/env cwl-runner {% comment %} Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} - -{% include 'notebox_begin' %} -The Arvados API and Git servers require Git 1.7.10 or later. -{% include 'notebox_end' %} +cwlVersion: v1.0 +class: CommandLineTool +inputs: [] +outputs: [] +arguments: ["echo", "hello world!"] diff --git a/doc/_includes/_tutorial_submit_job.liquid b/doc/_includes/_tutorial_submit_job.liquid deleted file mode 100644 index 548a6196a4..0000000000 --- a/doc/_includes/_tutorial_submit_job.liquid +++ /dev/null @@ -1,25 +0,0 @@ -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ - "name":"My md5 pipeline", - "components":{ - "do_hash":{ - "repository":"$USER/$USER", - "script":"hash.py", - "script_version":"master", - "runtime_constraints":{ - "docker_image":"arvados/jobs" - }, - "script_parameters":{ - "input":{ - "required": true, - "dataclass": "Collection" - } - } - } - } -} diff --git a/doc/_includes/_what_is_cwl.liquid b/doc/_includes/_what_is_cwl.liquid index fea11b3a32..7fbe7b8156 100644 --- a/doc/_includes/_what_is_cwl.liquid +++ b/doc/_includes/_what_is_cwl.liquid @@ -4,4 +4,4 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -The "Common Workflow Language (CWL)":http://commonwl.org is a multi-vendor open standard for describing analysis tools and workflows that are portable across a variety of platforms. CWL is the primary way to develop and run workflows for Arvados. Arvados supports versions "v1.0":http://commonwl.org/v1.0 and "v1.1":http://commonwl.org/v1.1 of the CWL specification. +The "Common Workflow Language (CWL)":http://commonwl.org is a multi-vendor open standard for describing analysis tools and workflows that are portable across a variety of platforms. CWL is the primary way to develop and run workflows for Arvados. Arvados supports versions "v1.0":http://commonwl.org/v1.0 , "v1.1":http://commonwl.org/v1.1 and "v1.2":http://commonwl.org/v1.2 of the CWL standard. diff --git a/doc/architecture/federation.html.textile.liquid b/doc/architecture/federation.html.textile.liquid index e2b80de707..2ec2c93fb4 100644 --- a/doc/architecture/federation.html.textile.liquid +++ b/doc/architecture/federation.html.textile.liquid @@ -26,7 +26,7 @@ Clusters are identified by a five-digit alphanumeric id (numbers and lowercase l Cluster identifiers are mapped API server hosts one of two ways: -* Through DNS resolution, under the @arvadosapi.com@ domain. For example, the API server for the cluster @qr1hi@ can be found at @qr1hi.arvadosapi.com@. To register a cluster id for free under @arvadosapi.com@, contact "info@curii.com":mailto:info@curii.com +* Through DNS resolution, under the @arvadosapi.com@ domain. For example, the API server for the cluster @pirca@ can be found at @pirca.arvadosapi.com@. To register a cluster id for free under @arvadosapi.com@, contact "info@curii.com":mailto:info@curii.com * Through explicit configuration: The @RemoteClusters@ section of @/etc/arvados/config.yml@ (for arvados-controller) diff --git a/doc/examples/pipeline_templates/gatk-exome-fq-snp.json b/doc/examples/pipeline_templates/gatk-exome-fq-snp.json deleted file mode 100644 index 481dda38ca..0000000000 --- a/doc/examples/pipeline_templates/gatk-exome-fq-snp.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "name":"GATK / exome PE fastq to snp", - "components":{ - "extract-reference":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"file-select", - "script_parameters":{ - "names":[ - "human_g1k_v37.fasta.gz", - "human_g1k_v37.fasta.fai.gz", - "human_g1k_v37.dict.gz" - ], - "input":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi" - }, - "output_name":false - }, - "bwa-index":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"bwa-index", - "script_parameters":{ - "input":{ - "output_of":"extract-reference" - }, - "bwa_tbz":{ - "value":"8b6e2c4916133e1d859c9e812861ce13+70", - "required":true - } - }, - "output_name":false - }, - "bwa-aln":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"bwa-aln", - "script_parameters":{ - "input":{ - "dataclass":"Collection", - "required":"true" - }, - "reference_index":{ - "output_of":"bwa-index" - }, - "samtools_tgz":{ - "value":"c777e23cf13e5d5906abfdc08d84bfdb+74", - "required":true - }, - "bwa_tbz":{ - "value":"8b6e2c4916133e1d859c9e812861ce13+70", - "required":true - } - }, - "runtime_constraints":{ - "max_tasks_per_node":1 - }, - "output_name":false - }, - "picard-gatk2-prep":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"picard-gatk2-prep", - "script_parameters":{ - "input":{ - "output_of":"bwa-aln" - }, - "reference":{ - "output_of":"extract-reference" - }, - "picard_zip":{ - "value":"687f74675c6a0e925dec619cc2bec25f+77", - "required":true - } - }, - "runtime_constraints":{ - "max_tasks_per_node":1 - }, - "output_name":false - }, - "GATK2-realign":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"GATK2-realign", - "script_parameters":{ - "input":{ - "output_of":"picard-gatk2-prep" - }, - "gatk_bundle":{ - "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi", - "required":true - }, - "picard_zip":{ - "value":"687f74675c6a0e925dec619cc2bec25f+77", - "required":true - }, - "gatk_tbz":{ - "value":"7e0a277d6d2353678a11f56bab3b13f2+87", - "required":true - }, - "regions":{ - "value":"13b53dbe1ec032dfc495fd974aa5dd4a+87/S02972011_Covered_sort_merged.bed" - }, - "region_padding":{ - "value":10 - } - }, - "runtime_constraints":{ - "max_tasks_per_node":2 - }, - "output_name":false - }, - "GATK2-bqsr":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"GATK2-bqsr", - "script_parameters":{ - "input":{ - "output_of":"GATK2-realign" - }, - "gatk_bundle":{ - "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi", - "required":true - }, - "picard_zip":{ - "value":"687f74675c6a0e925dec619cc2bec25f+77", - "required":true - }, - "gatk_tbz":{ - "value":"7e0a277d6d2353678a11f56bab3b13f2+87", - "required":true - } - }, - "output_name":false - }, - "GATK2-merge-call":{ - "repository":"arvados", - "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153", - "script":"GATK2-merge-call", - "script_parameters":{ - "input":{ - "output_of":"GATK2-bqsr" - }, - "gatk_bundle":{ - "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi", - "required":true - }, - "picard_zip":{ - "value":"687f74675c6a0e925dec619cc2bec25f+77", - "required":true - }, - "gatk_tbz":{ - "value":"7e0a277d6d2353678a11f56bab3b13f2+87", - "required":true - }, - "regions":{ - "value":"13b53dbe1ec032dfc495fd974aa5dd4a+87/S02972011_Covered_sort_merged.bed" - }, - "region_padding":{ - "value":10 - }, - "GATK2_UnifiedGenotyper_args":{ - "default":[ - "-stand_call_conf", - "30.0", - "-stand_emit_conf", - "30.0", - "-dcov", - "200" - ] - } - }, - "output_name":"Variant calls from UnifiedGenotyper" - } - } -} diff --git a/doc/examples/pipeline_templates/rtg-fq-snp.json b/doc/examples/pipeline_templates/rtg-fq-snp.json deleted file mode 100644 index c951c4c433..0000000000 --- a/doc/examples/pipeline_templates/rtg-fq-snp.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name":"Real Time Genomics / PE fastq to snp", - "components":{ - "extract_reference":{ - "script":"file-select", - "script_parameters":{ - "names":[ - "human_g1k_v37.fasta.gz" - ], - "input":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi" - }, - "script_version":"4c1f8cd1431ece2ef11c130d48bb2edfd2f00ec2" - }, - "reformat_reference":{ - "script_version":"4c1f8cd1431ece2ef11c130d48bb2edfd2f00ec2", - "script":"rtg-fasta2sdf", - "script_parameters":{ - "input":{ - "output_of":"extract_reference" - }, - "rtg_binary_zip":"5d33618193f763b7dc3a3fdfa11d452e+95+K@qr1hi", - "rtg_license":{ - "optional":false - } - } - }, - "reformat_reads":{ - "script_version":"4c1f8cd1431ece2ef11c130d48bb2edfd2f00ec2", - "script":"rtg-fastq2sdf", - "script_parameters":{ - "input":{ - "optional":false - }, - "rtg_binary_zip":"5d33618193f763b7dc3a3fdfa11d452e+95+K@qr1hi", - "rtg_license":{ - "optional":false - } - } - }, - "map_reads":{ - "script_version":"4c1f8cd1431ece2ef11c130d48bb2edfd2f00ec2", - "script":"rtg-map", - "script_parameters":{ - "input":{ - "output_of":"reformat_reads" - }, - "reference":{ - "output_of":"reformat_reference" - }, - "rtg_binary_zip":"5d33618193f763b7dc3a3fdfa11d452e+95+K@qr1hi", - "rtg_license":{ - "optional":false - } - }, - "runtime_constraints":{ - "max_tasks_per_node":1 - } - }, - "report_snp":{ - "script_version":"4c1f8cd1431ece2ef11c130d48bb2edfd2f00ec2", - "script":"rtg-snp", - "script_parameters":{ - "input":{ - "output_of":"map_reads" - }, - "reference":{ - "output_of":"reformat_reference" - }, - "rtg_binary_zip":"5d33618193f763b7dc3a3fdfa11d452e+95+K@qr1hi", - "rtg_license":{ - "optional":false - } - } - } - } -} diff --git a/doc/examples/ruby/list-active-nodes.rb b/doc/examples/ruby/list-active-nodes.rb deleted file mode 100755 index a3eb20540b..0000000000 --- a/doc/examples/ruby/list-active-nodes.rb +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env ruby -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: CC-BY-SA-3.0 - -abort 'Error: Ruby >= 1.9.3 required.' if RUBY_VERSION < '1.9.3' - -require 'arvados' - -arv = Arvados.new(api_version: 'v1') -arv.node.list[:items].each do |node| - if node[:crunch_worker_state] != 'down' - ping_age = (Time.now - Time.parse(node[:last_ping_at])).to_i rescue -1 - puts "#{node[:uuid]} #{node[:crunch_worker_state]} #{ping_age}" - end -end diff --git a/doc/install/copy_pipeline_from_curoverse.html.textile.liquid b/doc/install/copy_pipeline_from_curoverse.html.textile.liquid deleted file mode 100644 index 2c2b3c466e..0000000000 --- a/doc/install/copy_pipeline_from_curoverse.html.textile.liquid +++ /dev/null @@ -1,68 +0,0 @@ ---- -layout: default -navsection: installguide -title: Copy pipeline from the Arvados Playground -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -This tutorial describes how to find and copy a publicly shared pipeline from the Arvados Playground. Please note that you can use similar steps to copy any template you can access from the Arvados Playground to your cluster. - -h3. Access a public pipeline in the Arvados Playground using Workbench - -the Arvados Playground provides access to some public data, which can be used to experience Arvados in action. Let's access a public pipeline and copy it to your cluster, so that you can run it in your environment. - -Start by visiting the "*Arvados Playground public projects page*":https://playground.arvados.org/projects/public. This page lists all the publicly accessible projects in this arvados installation. Click on one of these projects to open it. We will use "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq as the example in this tutorial. - -Once in the "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq project, click on the *Pipeline templates* tab. In the pipeline templates tab, you will see a template named *lobSTR v.3*. Click on the *Show* button to the left of this name. This will take to you to the "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu template page. - -Once in this page, you can take the *uuid* of this template from the address bar, which is *qr1hi-p5p6p-9pkaxt6qjnkxhhu*. Next, we will copy this template to your Arvados instance. - -h3. Copying a pipeline template from the Arvados Playground to your cluster - -As described above, navigate to the publicly shared pipeline template "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu on the Arvados Playground. We will now copy this template with uuid *qr1hi-p5p6p-9pkaxt6qjnkxhhu* to your cluster. - -{% include 'tutorial_expectations' %} - -We will use the Arvados *arv-copy* command to copy this template to your cluster. In order to use arv-copy, first you need to setup the source and destination cluster configuration files. Here, *qr1hi* would be the source cluster and your Arvados instance would be the *dst_cluster*. - -During this setup, if you have an account in the Arvados Playground, you can use "your access token":#using-your-token to create the source configuration file. If you do not have an account in the Arvados Playground, you can use the "anonymous access token":#using-anonymous-token for the source cluster configuration. - -h4(#using-anonymous-token). *Configuring source and destination setup files using anonymous access token* - -Configure the source and destination clusters as described in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html tutorial in user guide, while using *5vqmz9mik2ou2k9objb8pnyce8t97p6vocyaouqo3qalvpmjs5* as the API token for source configuration. - - -
~$ cd ~/.config/arvados
-~$ echo "ARVADOS_API_HOST=qr1hi.arvadosapi.com" >> qr1hi.conf
-~$ echo "ARVADOS_API_TOKEN=5vqmz9mik2ou2k9objb8pnyce8t97p6vocyaouqo3qalvpmjs5" >> qr1hi.conf
-
- - -You can now copy the pipeline template from *qr1hi* to *your cluster*. Replace *dst_cluster* with the *ClusterID* of your cluster. - - -
~$  arv-copy --no-recursive --src qr1hi --dst dst_cluster qr1hi-p5p6p-9pkaxt6qjnkxhhu
-
- - -*Note:* When you are using anonymous access token to copy the template, you will not be able to do a recursive copy since you will not be able to provide the dst-git-repo parameter. In order to perform a recursive copy of the template, you would need to use the Arvados API token from your account as explained in the "using your token":#using-your-token section below. - -h4(#using-your-token). *Configuring source and destination setup files using personal access token* - -If you already have an account in the Arvados Playground, you can follow the instructions in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html user guide to get your *Current token* for source and destination clusters, and use them to create the source *qr1hi.conf* and dst_cluster.conf configuration files. - -You can now copy the pipeline template from *qr1hi* to *your cluster* with or without recursion. Replace *dst_cluster* with the *ClusterID* of your cluster. - -*Non-recursive copy:* - -
~$  arv-copy --no-recursive --src qr1hi --dst dst_cluster qr1hi-p5p6p-9pkaxt6qjnkxhhu
- - -*Recursive copy:* - -
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial qr1hi-p5p6p-9pkaxt6qjnkxhhu
- diff --git a/doc/sdk/java-v2/example.html.textile.liquid b/doc/sdk/java-v2/example.html.textile.liquid index e73f968c8d..8d2fc2f4af 100644 --- a/doc/sdk/java-v2/example.html.textile.liquid +++ b/doc/sdk/java-v2/example.html.textile.liquid @@ -28,7 +28,7 @@ public class CollectionExample { public static void main(String[] argv) { ConfigProvider conf = ExternalConfigProvider.builder(). apiProtocol("https"). - apiHost("qr1hi.arvadosapi.com"). + apiHost("zzzzz.arvadosapi.com"). apiPort(443). apiToken("..."). build(); diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid index 75c51ee5a8..82741c3ea6 100644 --- a/doc/sdk/python/cookbook.html.textile.liquid +++ b/doc/sdk/python/cookbook.html.textile.liquid @@ -47,7 +47,7 @@ h2. Get input of a CWL workflow {% codeblock as python %} import arvados api = arvados.api() -container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz" +container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"]) {% endcodeblock %} @@ -58,7 +58,7 @@ h2. Get output of a CWL workflow import arvados import arvados.collection api = arvados.api() -container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz" +container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() collection = arvados.collection.CollectionReader(container_request["output_uuid"]) print(collection.open("cwl.output.json").read()) @@ -89,7 +89,7 @@ def get_cr_state(cr_uuid): elif c['runtime_status'].get('warning', None): return 'Warning' return c['state'] -container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz' +container_request_uuid = 'zzzzz-xvhdp-zzzzzzzzzzzzzzz' print(get_cr_state(container_request_uuid)) {% endcodeblock %} @@ -98,7 +98,7 @@ h2. List input of child requests {% codeblock as python %} import arvados api = arvados.api() -parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" +parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] @@ -117,7 +117,7 @@ h2. List output of child requests {% codeblock as python %} import arvados api = arvados.api() -parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" +parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] @@ -136,7 +136,7 @@ h2. List failed child requests {% codeblock as python %} import arvados api = arvados.api() -parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" +parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] child_requests = api.container_requests().list(filters=[ @@ -155,7 +155,7 @@ h2. Get log of a child request import arvados import arvados.collection api = arvados.api() -container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" +container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() collection = arvados.collection.CollectionReader(container_request["log_uuid"]) for c in collection: @@ -169,7 +169,7 @@ h2(#sharing_link). Create a collection sharing link import arvados api = arvados.api() download="https://your.download.server" -collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz" +collection_uuid="zzzzz-4zz18-zzzzzzzzzzzzzzz" token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [ "GET /arvados/v1/collections/%s" % collection_uuid, "GET /arvados/v1/collections/%s/" % collection_uuid, @@ -185,8 +185,8 @@ Note, if two collections have files of the same name, the contents will be conca import arvados import arvados.collection api = arvados.api() -project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz" -collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"] +project_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz" +collection_uuids = ["zzzzz-4zz18-aaaaaaaaaaaaaaa", "zzzzz-4zz18-bbbbbbbbbbbbbbb"] combined_manifest = "" for u in collection_uuids: c = api.collections().get(uuid=u).execute() @@ -201,7 +201,7 @@ h2. Upload a file into a new collection import arvados import arvados.collection -project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz" +project_uuid = "zzzzz-j7d0g-zzzzzzzzzzzzzzz" collection_name = "My collection" filename = "file1.txt" @@ -223,7 +223,7 @@ h2. Download a file from a collection import arvados import arvados.collection -collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz" +collection_uuid = "zzzzz-4zz18-zzzzzzzzzzzzzzz" filename = "file1.txt" api = arvados.api() diff --git a/doc/sdk/python/events.html.textile.liquid b/doc/sdk/python/events.html.textile.liquid index afbec20d95..302af20c83 100644 --- a/doc/sdk/python/events.html.textile.liquid +++ b/doc/sdk/python/events.html.textile.liquid @@ -2,7 +2,7 @@ layout: default navsection: sdk navmenu: Python -title: Subscribing to events +title: Subscribing to database events ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. diff --git a/doc/sdk/ruby/example.html.textile.liquid b/doc/sdk/ruby/example.html.textile.liquid index b8c0dcbb80..f2ea1c09df 100644 --- a/doc/sdk/ruby/example.html.textile.liquid +++ b/doc/sdk/ruby/example.html.textile.liquid @@ -55,7 +55,7 @@ first_repo = repos[:items][0] puts "UUID of first repo returned is #{first_repo[:uuid]}" {% endcodeblock %} -UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz +UUID of first repo returned is zzzzz-s0uqq-b1bnybpx3u5temz h2. update diff --git a/doc/start/getting_started/firstpipeline.html.textile.liquid b/doc/start/getting_started/firstpipeline.html.textile.liquid deleted file mode 100644 index 43369a3bbf..0000000000 --- a/doc/start/getting_started/firstpipeline.html.textile.liquid +++ /dev/null @@ -1,94 +0,0 @@ ---- -layout: default -navsection: start -title: Run your first pipeline in minutes -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -h2. LobSTR v3 - -In this quickstart guide, we'll run an existing pipeline with pre-existing data. Step-by-step instructions are shown below. You can follow along using your own local install or by using the Arvados Playground (any Google account can be used to log in). - -(For more information about this pipeline, see our detailed lobSTR guide). - - - -Tip: You may need to make your browser window bigger to see full-size images in the gallery above. diff --git a/doc/start/getting_started/nextsteps.html.textile.liquid b/doc/start/getting_started/nextsteps.html.textile.liquid deleted file mode 100644 index dd059ea8d4..0000000000 --- a/doc/start/getting_started/nextsteps.html.textile.liquid +++ /dev/null @@ -1,12 +0,0 @@ ---- -layout: default -navsection: start -title: Check out the User Guide -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -Now that you've finished the Getting Started guide, check out the "User Guide":{{site.baseurl}}/user/index.html. The User Guide goes into more depth than the Getting Started guide, covers how to develop your own pipelines in addition to using pre-existing pipelines, covers the Arvados command line tools in addition to the Workbench graphical interface to Arvados, and can be referenced in any order. diff --git a/doc/start/getting_started/publicproject.html.textile.liquid b/doc/start/getting_started/publicproject.html.textile.liquid deleted file mode 100644 index 0fabad7aa7..0000000000 --- a/doc/start/getting_started/publicproject.html.textile.liquid +++ /dev/null @@ -1,133 +0,0 @@ ---- -layout: default -navsection: start -title: Visit an Arvados Public Project -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -h2. Mason Lab - Pathomap / Ancestry Mapper (Public) - -You can see Arvados in action by accessing the Mason Lab - Pathomap / Ancestry Mapper (Public) project. By visiting this project, you can see what an Arvados project is, access data collections in this project, and click through a pipeline instance's contents. - -You will be accessing this project in read-only mode and will not be able to make any modifications such as running a new pipeline instance. - - - -Tip: You may need to make your browser window bigger to see full-size images in the gallery above. diff --git a/doc/start/getting_started/sharedata.html.textile.liquid b/doc/start/getting_started/sharedata.html.textile.liquid deleted file mode 100644 index 02e0b70329..0000000000 --- a/doc/start/getting_started/sharedata.html.textile.liquid +++ /dev/null @@ -1,102 +0,0 @@ ---- -layout: default -navsection: start -title: Sharing Data -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -You can easily share data entirely through Workbench, the web interface to Arvados. - -h2. Upload and share your existing data - -Step-by-step instructions are shown below. - - - -Tip: You may need to make your browser window bigger to see full-size images in the gallery above. diff --git a/doc/start/index.html.textile.liquid b/doc/start/index.html.textile.liquid deleted file mode 100644 index cddfb8e441..0000000000 --- a/doc/start/index.html.textile.liquid +++ /dev/null @@ -1,133 +0,0 @@ ---- -layout: default -navsection: start -title: Welcome to Arvados! -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -This guide provides an introduction to using Arvados to solve big data bioinformatics problems. - -h2. What is Arvados? - -Arvados is a free and open source bioinformatics platform for genomic and biomedical data. - -We address the needs of IT directors, lab principals, and bioinformaticians. - -h2. Why use Arvados? - -Arvados enables you to quickly begin using cloud computing resources in your bioinformatics work. It allows you to track your methods and datasets, share them securely, and easily re-run analyses. - -h3. Take a look (Screenshots gallery) - - - -Note: Workbench is the web interface to Arvados. -Tip: You may need to make your browser window bigger to see full-size images in the gallery above. - -h3. Key Features - - diff --git a/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml b/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml index 73bd9f599c..73dd65c463 100755 --- a/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml +++ b/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml @@ -15,15 +15,15 @@ cwl:tool: bwa-mem.cwl reference: class: File location: keep:2463fa9efeb75e099685528b3b9071e0+438/19.fasta.bwt - arv:collectionUUID: qr1hi-4zz18-pwid4w22a40jp8l + arv:collectionUUID: jutro-4zz18-tv416l321i4r01e read_p1: class: File location: keep:ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_1.fastq - arv:collectionUUID: qr1hi-4zz18-h615rgfmqt3wje0 + arv:collectionUUID: jutro-4zz18-8k5hsvee0izv2g3 read_p2: class: File location: keep:ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_2.fastq - arv:collectionUUID: qr1hi-4zz18-h615rgfmqt3wje0 + arv:collectionUUID: jutro-4zz18-8k5hsvee0izv2g3 group_id: arvados_tutorial sample_id: HWI-ST1027_129 PL: illumina diff --git a/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml b/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml index 7e71e959eb..e76aa78173 100755 --- a/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml +++ b/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml @@ -9,13 +9,13 @@ cwl:tool: bwa-mem.cwl reference: class: File - location: keep:qr1hi-4zz18-pwid4w22a40jp8l/19.fasta.bwt + location: keep:jutro-4zz18-tv416l321i4r01e/19.fasta.bwt read_p1: class: File - location: keep:qr1hi-4zz18-h615rgfmqt3wje0/HWI-ST1027_129_D0THKACXX.1_1.fastq + location: keep:jutro-4zz18-8k5hsvee0izv2g3/HWI-ST1027_129_D0THKACXX.1_1.fastq read_p2: class: File - location: keep:qr1hi-4zz18-h615rgfmqt3wje0/HWI-ST1027_129_D0THKACXX.1_2.fastq + location: keep:jutro-4zz18-8k5hsvee0izv2g3/HWI-ST1027_129_D0THKACXX.1_2.fastq group_id: arvados_tutorial sample_id: HWI-ST1027_129 PL: illumina diff --git a/doc/user/cwl/bwa-mem/bwa-mem.cwl b/doc/user/cwl/bwa-mem/bwa-mem.cwl index 2001971264..018867c83e 100755 --- a/doc/user/cwl/bwa-mem/bwa-mem.cwl +++ b/doc/user/cwl/bwa-mem/bwa-mem.cwl @@ -8,13 +8,13 @@ class: CommandLineTool hints: DockerRequirement: - dockerPull: lh3lh3/bwa + dockerPull: quay.io/biocontainers/bwa:0.7.17--ha92aebf_3 -baseCommand: [mem] +baseCommand: [bwa, mem] arguments: - {prefix: "-t", valueFrom: $(runtime.cores)} - - {prefix: "-R", valueFrom: "@RG\tID:$(inputs.group_id)\tPL:$(inputs.PL)\tSM:$(inputs.sample_id)"} + - {prefix: "-R", valueFrom: '@RG\\\tID:$(inputs.group_id)\\\tPL:$(inputs.PL)\\\tSM:$(inputs.sample_id)'} inputs: reference: diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid index 725528f44d..761d198ee4 100644 --- a/doc/user/cwl/cwl-run-options.html.textile.liquid +++ b/doc/user/cwl/cwl-run-options.html.textile.liquid @@ -1,7 +1,7 @@ --- layout: default navsection: userguide -title: "Using arvados-cwl-runner" +title: "arvados-cwl-runner options" ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. @@ -74,10 +74,10 @@ Use the @--name@ and @--output-name@ options to specify the name of the workflow 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "Example bwa run" --output-name "Example bwa output" bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -98,9 +98,9 @@ To submit a workflow and exit immediately, use the @--no-wait@ option.  This wil
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --no-wait bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Uploaded to qr1hi-4zz18-eqnfwrow8aysa9q
-2016-06-30 15:07:52 arvados.cwl-runner[12480] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-qr1hi-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Uploaded to zzzzz-4zz18-eqnfwrow8aysa9q
+2016-06-30 15:07:52 arvados.cwl-runner[12480] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+zzzzz-8i9sb-fm2n3b1w0l6bskg
 

 
 
@@ -111,10 +111,10 @@ To run a workflow with local control, use @--local@.  This means that the host w
 
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --local bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 10:05:19 arvados.cwl-runner[16290] INFO: Pipeline instance qr1hi-d1hrv-92wcu6ldtio74r4
-2016-07-01 10:05:28 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Queued
-2016-07-01 10:05:29 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Running
-2016-07-01 10:05:45 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Complete
+2016-07-01 10:05:19 arvados.cwl-runner[16290] INFO: Pipeline instance zzzzz-d1hrv-92wcu6ldtio74r4
+2016-07-01 10:05:28 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Queued
+2016-07-01 10:05:29 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Running
+2016-07-01 10:05:45 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Complete
 2016-07-01 10:05:46 arvados.cwl-runner[16290] INFO: Overall process status is success
 {
     "aligned_sam": {
diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid
index 2be803b52a..442a60b04f 100644
--- a/doc/user/cwl/cwl-runner.html.textile.liquid
+++ b/doc/user/cwl/cwl-runner.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Running an Arvados workflow"
+title: "Starting a Workflow at the Command Line"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -13,44 +13,38 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 {% include 'tutorial_expectations' %}
 
-{% include 'notebox_begin' %}
-
-By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If you want to submit jobs from somewhere else, such as your workstation, you may install "arvados-cwl-runner.":#setup
-
-{% include 'notebox_end' %}
-
 This tutorial will demonstrate how to submit a workflow at the command line using @arvados-cwl-runner@.
 
-h2. Running arvados-cwl-runner
+# "Get the tutorial files":#get-files
+# "Submitting a workflow to an Arvados cluster":#submitting
+# "Registering a workflow to use in Workbench":#registering
+# "Make a workflow file directly executable":#executable
 
-h3. Get the example files
+h2(#get-files). Get the tutorial files
 
-The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem
+The tutorial files are located in the documentation section of the Arvados source repository, which can be found on "git.arvados.org":https://git.arvados.org/arvados.git/tree/HEAD:/doc/user/cwl/bwa-mem or "github":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem
 
 
-
~$ git clone https://github.com/arvados/arvados
+
~$ git clone https://git.arvados.org/arvados.git
 ~$ cd arvados/doc/user/cwl/bwa-mem
 

 
 
-The tutorial data is hosted on "https://playground.arvados.org":https://playground.arvados.org (also referred to by the identifier *qr1hi*).  If you are using a different Arvados instance, you may need to copy the data to your own instance.  The easiest way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free playground.arvados.org account).
+The tutorial data is hosted on "https://playground.arvados.org":https://playground.arvados.org (also referred to by the identifier *pirca*).  If you are using a different Arvados instance, you may need to copy the data to your own instance.  One way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free playground.arvados.org account).
 
 
-
~$ arv-copy --src qr1hi --dst settings 2463fa9efeb75e099685528b3b9071e0+438
-~$ arv-copy --src qr1hi --dst settings ae480c5099b81e17267b7445e35b4bc7+180
-~$ arv-copy --src qr1hi --dst settings 655c6cd07550151b210961ed1d3852cf+57
+
~$ arv-copy --src pirca --dst settings 2463fa9efeb75e099685528b3b9071e0+438
+~$ arv-copy --src pirca --dst settings ae480c5099b81e17267b7445e35b4bc7+180
 

 
 
 If you do not wish to create an account on "https://playground.arvados.org":https://playground.arvados.org, you may download the files anonymously and upload them to your local Arvados instance:
 
-"https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438":https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438
-
-"https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180":https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180
+"https://collections.pirca.arvadosapi.com/c=2463fa9efeb75e099685528b3b9071e0+438/":https://collections.pirca.arvadosapi.com/c=2463fa9efeb75e099685528b3b9071e0+438/
 
-"https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57":https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57
+"https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/":https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/
 
-h2. Submitting a workflow to an Arvados cluster
+h2(#submitting). Submitting a workflow to an Arvados cluster
 
 h3. Submit a workflow and wait for results
 
@@ -62,10 +56,10 @@ Use @arvados-cwl-runner@ to submit CWL workflows to Arvados.  After submitting t
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -88,15 +82,19 @@ If you reference a file in "arv-mount":{{site.baseurl}}/user/tutorials/tutorial-
 
 If you reference a local file which is not in @arv-mount@, then @arvados-cwl-runner@ will upload the file to Keep and use the Keep URI reference from the upload.
 
-You can also execute CWL files directly from Keep:
+You can also execute CWL files that have been uploaded Keep:
 
 
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl bwa-mem-input.yml
+

+~/arvados/doc/user/cwl/bwa-mem$ arv-put --portable-data-hash --name "bwa-mem.cwl" bwa-mem.cwl
+2020-08-20 13:40:02 arvados.arv_put[12976] INFO: Collection saved as 'bwa-mem.cwl'
+f141fc27e7cfa7f7b6d208df5e0ee01b+59
+~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:f141fc27e7cfa7f7b6d208df5e0ee01b+59/bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -109,50 +107,128 @@ arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107,
 

 
 
+Note: uploading a workflow file to Keep is _not_ the same as registering the workflow for use in Workbench.  See "Registering a workflow to use in Workbench":#registering below.
+
 h3. Work reuse
 
 Workflows submitted with @arvados-cwl-runner@ will take advantage of Arvados job reuse.  If you submit a workflow which is identical to one that has run before, it will short cut the execution and return the result of the previous run.  This also applies to individual workflow steps.  For example, a two step workflow where the first step has run before will reuse results for first step and only execute the new second step.  You can disable this behavior with @--disable-reuse@.
 
 h3. Command line options
 
-See "Using arvados-cwl-runner":{{site.baseurl}}/user/cwl/cwl-run-options.html
+See "arvados-cwl-runner options":{{site.baseurl}}/user/cwl/cwl-run-options.html
 
-h2(#setup). Setting up arvados-cwl-runner
+h2(#registering). Registering a workflow to use in Workbench
 
-By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If you want to submit jobs from somewhere else, such as your workstation, you may install @arvados-cwl-runner@ using @pip@:
+Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the  Run a process... button on the Workbench Dashboard and on the command line by UUID.
 
 
-
~$ virtualenv ~/venv
-~$ . ~/venv/bin/activate
-~$ pip install -U setuptools
-~$ pip install arvados-cwl-runner
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
+2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template zzzzz-p5p6p-rjleou1dwr167v5
+zzzzz-p5p6p-rjleou1dwr167v5
 

 
 
-h3. Check Docker access
+You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
 
-In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Arvados.
+
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to zzzzz-4zz18-0f91qkovk4ml18o
+2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template zzzzz-p5p6p-0deqe6nuuyqns2i
+zzzzz-p5p6p-zuniv58hn8d0qd8
+

+
 
-You can determine if you have access to Docker by running @docker version@:
+h3. Running registered workflows at the command line
+
+You can run a registered workflow at the command line by its UUID:
 
 
-
~$ docker version
-Client:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
-
-Server:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner pirca-7fd4e-3nqbw08vtjl8ybz --help
+INFO /home/peter/work/scripts/venv3/bin/arvados-cwl-runner 2.1.0.dev20200814195416, arvados-python-client 2.1.0.dev20200814195416, cwltool 3.0.20200807132242
+INFO Resolved 'pirca-7fd4e-3nqbw08vtjl8ybz' to 'arvwf:pirca-7fd4e-3nqbw08vtjl8ybz#main'
+usage: pirca-7fd4e-3nqbw08vtjl8ybz [-h] [--PL PL] [--group_id GROUP_ID]
+                                   [--read_p1 READ_P1] [--read_p2 READ_P2]
+                                   [--reference REFERENCE]
+                                   [--sample_id SAMPLE_ID]
+                                   [job_order]
+
+positional arguments:
+  job_order             Job input json file
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --PL PL
+  --group_id GROUP_ID
+  --read_p1 READ_P1     The reads, in fastq format.
+  --read_p2 READ_P2     For mate paired reads, the second file (optional).
+  --reference REFERENCE
+                        The index files produced by `bwa index`
+  --sample_id SAMPLE_ID
 

 
 
-If this returns an error, contact the sysadmin of your cluster for assistance.
+h2(#executable). Make a workflow file directly executable
+
+You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file:
+
+
+
#!/usr/bin/env cwl-runner
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+You can even make an input file directly executable the same way with the following two lines at the top:
+
+
+
#!/usr/bin/env cwl-runner
+cwl:tool: bwa-mem.cwl
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+h2(#setup). Setting up arvados-cwl-runner
+
+See "Arvados CWL Runner":{{site.baseurl}}/sdk/python/arvados-cwl-runner.html
diff --git a/doc/user/cwl/cwl-style.html.textile.liquid b/doc/user/cwl/cwl-style.html.textile.liquid
index ee36014cb5..bd07161ce3 100644
--- a/doc/user/cwl/cwl-style.html.textile.liquid
+++ b/doc/user/cwl/cwl-style.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: Writing Portable High-Performance Workflows
+title: Guidelines for Writing High-Performance Portable Workflows
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
diff --git a/doc/user/cwl/cwl-versions.html.textile.liquid b/doc/user/cwl/cwl-versions.html.textile.liquid
index 5fcfcbe3bc..ac679dc154 100644
--- a/doc/user/cwl/cwl-versions.html.textile.liquid
+++ b/doc/user/cwl/cwl-versions.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: CWL version and API support
+title: CWL version support
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,6 +9,8 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
+Arvados supports CWL v1.0, v1.1 and v1.2.
+
 h2(#v12). Upgrading your workflows to CWL v1.2
 
 If you are starting from a CWL v1.0 document, see "Upgrading your workflows to CWL v1.1":#v11 below.
diff --git a/doc/user/getting_started/check-environment.html.textile.liquid b/doc/user/getting_started/check-environment.html.textile.liquid
index b707891a1e..1097e4e9d8 100644
--- a/doc/user/getting_started/check-environment.html.textile.liquid
+++ b/doc/user/getting_started/check-environment.html.textile.liquid
@@ -16,14 +16,14 @@ Check that you are able to access the Arvados API server using @arv user current
 
 
$ arv user current
 {
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/users/qr1hi-xioed-9z2p3pn12yqdaem",
+ "href":"https://zzzzz.arvadosapi.com/arvados/v1/users/zzzzz-xioed-9z2p3pn12yqdaem",
  "kind":"arvados#user",
  "etag":"8u0xwb9f3otb2xx9hto4wyo03",
- "uuid":"qr1hi-tpzed-92d3kxnimy3d4e8",
- "owner_uuid":"qr1hi-tpqed-23iddeohxta2r59",
+ "uuid":"zzzzz-tpzed-92d3kxnimy3d4e8",
+ "owner_uuid":"zzzzz-tpqed-23iddeohxta2r59",
  "created_at":"2013-12-02T17:05:47Z",
- "modified_by_client_uuid":"qr1hi-xxfg8-owxa2oa2s33jyej",
- "modified_by_user_uuid":"qr1hi-tpqed-23iddeohxta2r59",
+ "modified_by_client_uuid":"zzzzz-xxfg8-owxa2oa2s33jyej",
+ "modified_by_user_uuid":"zzzzz-tpqed-23iddeohxta2r59",
  "modified_at":"2013-12-02T17:07:08Z",
  "updated_at":"2013-12-05T19:51:08Z",
  "email":"you@example.com",
diff --git a/doc/user/getting_started/ssh-access-unix.html.textile.liquid b/doc/user/getting_started/ssh-access-unix.html.textile.liquid
index 284d0a1f04..80cb391314 100644
--- a/doc/user/getting_started/ssh-access-unix.html.textile.liquid
+++ b/doc/user/getting_started/ssh-access-unix.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This document is for accessing an Arvados VM using SSH keys in Unix environments (Linux, OS X, Cygwin). If you would like to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Windows environment, please visit the "Accessing an Arvados VM with SSH - Windows Environments":ssh-access-windows.html page.
+This document is for accessing an Arvados VM using SSH keys in Unix-like environments (Linux, macOS, Cygwin, Windows Subsystem for Linux). If you would like to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Windows environment, please visit the "Accessing an Arvados VM with SSH - Windows Environments":ssh-access-windows.html page.
 
 {% include 'ssh_intro' %}
 
@@ -49,7 +49,7 @@ ssh-rsa AAAAB3NzaC1ycEDoNotUseExampleKeyDoNotUseExampleKeyDoNotUseExampleKeyDoNo
 
 Now you can set up @ssh-agent@ (next) or proceed with "adding your key to the Arvados Workbench.":#workbench
 
-h3. Set up ssh-agent (recommended)
+h3. Set up ssh-agent (optional)
 
 If you find you are entering your passphrase frequently, you can use @ssh-agent@ to manage your credentials.  Use @ssh-add -l@ to test if you already have ssh-agent running:
 
@@ -80,11 +80,21 @@ When everything is set up, @ssh-add -l@ should yield output that looks something
 
 {% include 'ssh_addkey' %}
 
-h3. Connecting to the virtual machine
+h3. Connecting directly
 
-Use the following command to connect to the _shell_ VM instance as _you_.  Replace *you@shell* at the end of the following command with your *login* and *hostname* from Workbench:
+If the VM is available on the public Internet (or you are on the same private network as the VM) you can connect directly with @ssh@.  You can probably copy-and-paste the text from *Command line* column directly into a terminal.
 
-notextile. 
$ ssh -o "ProxyCommand ssh -p2222 turnout@switchyard.{{ site.arvados_api_host }} -x -a shell" -x you@shell

+Use the following example command to connect as _you_ to the _shell.ClusterID.example.com_ VM instance.  Replace *you@shell.ClusterID.example.com* at the end of the following command with your *login* and *hostname* from Workbench.
+
+notextile. 
$ ssh you@shell.ClusterID.example.com

+
+h3. Connecting through switchyard
+
+Some Arvados installations use "switchyard" to isolate shell VMs from the public Internet.
+
+Use the following example command to connect to the _shell_ VM instance as _you_.  Replace *you@shell* at the end of the following command with your *login* and *hostname* from Workbench:
+
+notextile. 
$ ssh -o "ProxyCommand ssh -p2222 turnout@switchyard.ClusterID.example.com -x -a shell" -x you@shell

 
 This command does several things at once. You usually cannot log in directly to virtual machines over the public Internet.  Instead, you log into a "switchyard" server and then tell the switchyard which virtual machine you want to connect to.
 
@@ -99,7 +109,7 @@ This command does several things at once. You usually cannot log in directly to
 
 You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
 
-h3. Configuration (recommended)
+h4. Configuration (recommended)
 
 The command line above is cumbersome, but you can configure SSH to remember many of these settings.  Add this text to the file @.ssh/config@ in your home directory (create a new file if @.ssh/config@ doesn't exist):
 
diff --git a/doc/user/getting_started/ssh-access-windows.html.textile.liquid b/doc/user/getting_started/ssh-access-windows.html.textile.liquid
index 0406e7c03b..5cbe2a3285 100644
--- a/doc/user/getting_started/ssh-access-windows.html.textile.liquid
+++ b/doc/user/getting_started/ssh-access-windows.html.textile.liquid
@@ -9,13 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This document is for accessing an Arvados VM using SSH keys in Windows environments. If you would like to use to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Unix environment (Linux, OS X, Cygwin), please visit the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.
+This document is for accessing an Arvados VM using SSH keys in Windows environments using PuTTY.  If you would like to use to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page.  If you are using a Unix-like environment (Linux, macOS, Cygwin, or Windows Subsystem for Linux), please visit the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.
 
 {% include 'ssh_intro' %}
 
 h1(#gettingkey). Getting your SSH key
 
-(Note: if you are using the SSH client that comes with "Cygwin":http://cygwin.com, please use instructions found in the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.)
+(Note: If you are using the SSH client that comes with "Cygwin":http://cygwin.com or Windows Subsystem for Linux (WSL) please use instructions found in the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.)
 
 We will be using PuTTY to connect to Arvados. "PuTTY":http://www.chiark.greenend.org.uk/~sgtatham/putty/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTY includes all the tools a Windows user needs to create private keys and make SSH connections to your virtual machines in the Arvados Cloud.
 
@@ -57,6 +57,16 @@ Pageant is a PuTTY utility that manages your private keys so is not necessary to
 
 h3. Initial configuration
 
+h4. Connecting directly
+
+# Open PuTTY from the Start Menu.
+# On the Session screen set the Host Name (or IP address) to “shell.ClusterID.example.com”, which is the hostname listed in the _Virtual Machines_ page.
+# On the Session screen set the Port to “22”.
+# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *Login name* column on the Arvados Workbench Virtual machines_ page.
+# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and click Save.
+
+h4. Connecting through switchyard
+
 # Open PuTTY from the Start Menu.
 # On the Session screen set the Host Name (or IP address) to “shell”, which is the hostname listed in the _Virtual Machines_ page.
 # On the Session screen set the Port to “22”.
diff --git a/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid b/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
index 551002e55e..2aa494ae9f 100644
--- a/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
+++ b/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
@@ -15,6 +15,10 @@ h2(#webshell). Access VM using webshell
 
 Webshell gives you access to an arvados virtual machine from your browser with no additional setup.
 
+{% include 'notebox_begin' %}
+Some Arvados clusters may not have webshell set up.  If you do not see a "Log in" button or "web shell" column, you will have to follow the "Unix":ssh-access-unix.html or "Windows":ssh-access-windows.html @ssh@ instructions.
+{% include 'notebox_end' %}
+
 In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Virtual machines* to see the list of virtual machines you can access.  If you do not have access to any virtual machines, please click on Send request for shell access or send an email to "support@curoverse.com":mailto:support@curoverse.com.
 
 Each row in the Virtual Machines panel lists the hostname of the VM, along with a Log in as *you* button under the column "Web shell". Clicking on this button will open up a webshell terminal for you in a new browser tab and log you in.
diff --git a/doc/user/getting_started/workbench.html.textile.liquid b/doc/user/getting_started/workbench.html.textile.liquid
index fc704227e0..e8f76b6260 100644
--- a/doc/user/getting_started/workbench.html.textile.liquid
+++ b/doc/user/getting_started/workbench.html.textile.liquid
@@ -9,14 +9,20 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-If you are using the default Arvados instance for this guide, you can Access Arvados Workbench using this link:
+{% include 'notebox_begin' %}
+This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1".  There is also a new Workbench web application under development called "Workbench 2".  Sites which have both Workbench applications installed will have a dropdown menu option "Switch to Workbench 2" to switch between versions.
+
+This guide will be updated to cover "Workbench 2" in the future.
+{% include 'notebox_end' %}
+
+If you are using the "playground" Arvados instance for this guide, you can Access Arvados Workbench using this link:
 
 {{site.arvados_workbench_host}}/
 
 (If you are using a different Arvados instance than the default for this guide, replace *{{ site.arvados_workbench_host }}* with your private instance in all of the examples in this guide.)
 
-You may be asked to log in using a Google account.  Arvados uses only your name and email address from Google services for identification, and will never access any personal information.  If you are accessing Arvados for the first time, the Workbench may indicate your account status is *New / inactive*.  If this is the case, contact the administrator of the Arvados instance to request activation of your account.
+You will be asked to log in.  Arvados uses only your name and email address for identification, and will never access any personal information.  If you are accessing Arvados for the first time, the Workbench may indicate your account status is *New / inactive*.  If this is the case, contact the administrator of the Arvados instance to request activation of your account.
 
-Once your account is active, logging in to the Workbench will present you with the Dashboard. This gives a summary of your projects and recent activity in the Arvados instance.  "You are now ready to run your first pipeline.":{{ site.baseurl }}/user/tutorials/tutorial-workflow-workbench.html
+Once your account is active, logging in to the Workbench will present you with the Dashboard. This gives a summary of your projects and recent activity in the Arvados instance.  You are now ready to "upload data":{{ site.baseurl }}/user/tutorials/tutorial-keep.html or "run your first workflow.":{{ site.baseurl }}/user/tutorials/tutorial-workflow-workbench.html
 
 !{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/workbench-dashboard.png!
diff --git a/doc/user/index.html.textile.liquid b/doc/user/index.html.textile.liquid
index 909394ef47..9749d1f284 100644
--- a/doc/user/index.html.textile.liquid
+++ b/doc/user/index.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This guide provides a reference for using Arvados to solve scientific big data problems, including:
+Arvados is an open source platform for managing, processing, and sharing genomic and other large scientific and biomedical data.  This guide provides a reference for using Arvados to solve scientific big data problems, including:
 
 * Robust storage of very large files, such as whole genome sequences, using the "Arvados Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html content-addressable cluster file system.
 * Running compute-intensive scientific analysis pipelines, such as genomic alignment and variant calls using the "Arvados Crunch":{{site.baseurl}}/user/tutorials/intro-crunch.html cluster compute engine.
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 0f0e40be9c..d35df4fcec 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -9,103 +9,74 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-{% include 'crunch1only_begin' %}
-On those sites, the "copy a pipeline template" feature described below is not available. However, "copy a workflow" feature is not yet implemented.
-{% include 'crunch1only_end' %}
-
 This tutorial describes how to copy Arvados objects from one cluster to another by using @arv-copy@.
 
 {% include 'tutorial_expectations' %}
 
 h2. arv-copy
 
-@arv-copy@ allows users to copy collections and pipeline templates from one cluster to another. By default, @arv-copy@ will recursively go through a template and copy all dependencies associated with the object.
+@arv-copy@ allows users to copy collections and workflows from one cluster to another. By default, @arv-copy@ will recursively go through the workflow and copy all dependencies associated with the object.
 
-For example, let's copy from the Arvados playground, also known as *qr1hi*, to *dst_cluster*. The names *qr1hi* and *dst_cluster* are interchangable with any cluster name. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *qr1hi*-4zz18-tci4vn4fa95w0zx, the cluster name is qr1hi.
+For example, let's copy from the Arvados playground, also known as *pirca*, to *dstcl*. The names *pirca* and *dstcl* are interchangable with any cluster id. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *zzzzz*-4zz18-tci4vn4fa95w0zx, the cluster name is *zzzzz* .
 
-In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files, one for each cluster. The names of the files must have the format of *ClusterID.conf*. In our example, let's make two files, one for *qr1hi* and one for *dst_cluster*. From your *Current token* page in *qr1hi* and *dst_cluster*, copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
+In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files in @~/.config/arvados@, one for each cluster. The names of the files must have the format of *ClusterID.conf*. Navigate to the *Current token* page on each of *pirca* and *dstcl* to get the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
 
 !{display: block;margin-left: 25px;margin-right: auto;}{{ site.baseurl }}/images/api-token-host.png!
 
-Copy your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ into the config files as shown below in the shell account from which you are executing the commands. For example, the default shell you may have access to is shell.qr1hi. You can add these files in ~/.config/arvados/ in the qr1hi shell terminal.
+The config file consists of two lines, one for ARVADOS_API_HOST and one for ARVADOS_API_TOKEN:
 
-
-
~$ cd ~/.config/arvados
-~$ echo "ARVADOS_API_HOST=qr1hi.arvadosapi.com" >> qr1hi.conf
-~$ echo "ARVADOS_API_TOKEN=123456789abcdefghijkl" >> qr1hi.conf
-~$ echo "ARVADOS_API_HOST=dst_cluster.arvadosapi.com" >> dst_cluster.conf
-~$ echo "ARVADOS_API_TOKEN=987654321lkjihgfedcba" >> dst_cluster.conf
-

-
+
+ARVADOS_API_HOST=zzzzz.arvadosapi.com
+ARVADOS_API_TOKEN=v2/zzzzz-gj3su-xxxxxxxxxxxxxxx/123456789abcdefghijkl
+

+
+Copy your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ into the config files as shown below in the shell account from which you are executing the commands.  In our example, you need two files, @~/.config/arvados/pirca.conf@ and @~/.config/arvados/dstcl.conf@.
 
-Now you're ready to copy between *qr1hi* and *dst_cluster*!
+Now you're ready to copy between *pirca* and *dstcl*!
 
 h3. How to copy a collection
 
-First, select the uuid of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
+First, determine the uuid or portable data hash of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
 
-Now copy the collection from *qr1hi* to *dst_cluster*. We will use the uuid @qr1hi-4zz18-tci4vn4fa95w0zx@ as an example. You can find this collection in the lobSTR v.3 project on playground.arvados.org.
+Now copy the collection from *pirca* to *dstcl*. We will use the uuid @jutro-4zz18-tv416l321i4r01e@ as an example. You can find this collection on playground.arvados.org.
 
-
~$ arv-copy --src qr1hi --dst dst_cluster qr1hi-4zz18-tci4vn4fa95w0zx
-qr1hi-4zz18-tci4vn4fa95w0zx: 6.1M / 6.1M 100.0%
-arvados.arv-copy[1234] INFO: Success: created copy with uuid dst_cluster-4zz18-8765943210cdbae
-

-
-
-The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in a pre-created project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
-
-For example, this will copy the collection to project dst_cluster-j7d0g-a894213ukjhal12 in the destination cluster.
-
- 
~$ arv-copy --src qr1hi --dst dst_cluster --project-uuid dst_cluster-j7d0g-a894213ukjhal12 qr1hi-4zz18-tci4vn4fa95w0zx
+
~$ arv-copy --src pirca --dst dstcl jutro-4zz18-tv416l321i4r01e
+jutro-4zz18-tv416l321i4r01e: 6.1M / 6.1M 100.0%
+arvados.arv-copy[1234] INFO: Success: created copy with uuid dstcl-4zz18-xxxxxxxxxxxxxxx
 

 
 
-h3. How to copy a pipeline template
-
-{% include 'arv_copy_expectations' %}
-
-We will use the uuid @qr1hi-p5p6p-9pkaxt6qjnkxhhu@ as an example pipeline template.
+You can also copy by content address:
 
 
-
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial qr1hi-p5p6p-9pkaxt6qjnkxhhu
-To git@git.dst_cluster.arvadosapi.com:$USER/tutorial.git
- * [new branch] git_git_qr1hi_arvadosapi_com_arvados_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d -> git_git_qr1hi_arvadosapi_com_arvados_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d
-arvados.arv-copy[19694] INFO: Success: created copy with uuid dst_cluster-p5p6p-rym2h5ub9m8ofwj
+
~$ arv-copy --src pirca --dst dstcl 2463fa9efeb75e099685528b3b9071e0+438
+2463fa9efeb75e099685528b3b9071e0+438: 6.1M / 6.1M 100.0%
+arvados.arv-copy[1234] INFO: Success: created copy with uuid dstcl-4zz18-xxxxxxxxxxxxxxx
 

 
 
-New branches in the destination git repo will be created for each branch used in the pipeline template. For example, if your source branch was named ac21f0d45a76294aaca0c0c0fdf06eb72d03368d, your new branch will be named @git_git_qr1hi_arvadosapi_com_reponame_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d@.
-
-By default, if you copy a pipeline template recursively, you will find that the template as well as all the dependencies are in your home project.
+The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in an existing project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
 
-If you would like to copy the object without dependencies, you can use the @--no-recursive@ tag.
+For example, this will copy the collection to project dstcl-j7d0g-a894213ukjhal12 in the destination cluster.
 
-For example, we can copy the same object using this tag.
-
-
-
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive qr1hi-p5p6p-9pkaxt6qjnkxhhu
+ 
~$ arv-copy --src pirca --dst dstcl --project-uuid dstcl-j7d0g-a894213ukjhal12 jutro-4zz18-tv416l321i4r01e
 

 
 
 h3. How to copy a workflow
 
-We will use the uuid @zzzzz-7fd4e-sampleworkflow1@ as an example workflow.
+We will use the uuid @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
 
 
-
~$ arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial zzzzz-7fd4e-sampleworkflow1
-zzzzz-4zz18-jidprdejysravcr: 1143M / 1143M 100.0%
-2017-01-04 04:11:58 arvados.arv-copy[5906] INFO:
-2017-01-04 04:11:58 arvados.arv-copy[5906] INFO: Success: created copy with uuid dst_cluster-7fd4e-ojtgpne594ubkt7
+
~$ arv-copy --src jutro --dst pirca --project-uuid pirca-j7d0g-ecak8knpefz8ere jutro-7fd4e-mkmmq53m1ze6apx
+ae480c5099b81e17267b7445e35b4bc7+180: 23M / 23M 100.0%
+2463fa9efeb75e099685528b3b9071e0+438: 156M / 156M 100.0%
+jutro-4zz18-vvvqlops0a0kpdl: 94M / 94M 100.0%
+2020-08-19 17:04:13 arvados.arv-copy[4789] INFO:
+2020-08-19 17:04:13 arvados.arv-copy[4789] INFO: Success: created copy with uuid pirca-7fd4e-s0tw9rfbkpo2fmx
 

 
 
-The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *locations* and *docker images* found in the src workflow definition will also be copied to the destination recursively.
+The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *collections* and *docker images* referenced in the source workflow definition will also be copied to the destination.
 
 If you would like to copy the object without dependencies, you can use the @--no-recursive@ flag.
-
-For example, we can copy the same object non-recursively using the following:
-
-
-
~$ arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive zzzzz-7fd4e-sampleworkflow1
-

-
diff --git a/doc/user/topics/arv-docker.html.textile.liquid b/doc/user/topics/arv-docker.html.textile.liquid
index e9e8450268..bb1c7dd53e 100644
--- a/doc/user/topics/arv-docker.html.textile.liquid
+++ b/doc/user/topics/arv-docker.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Customizing Crunch environment using Docker"
+title: "Working with Docker images"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,145 +9,80 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This page describes how to customize the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a crunch script will be run in using "Docker.":https://www.docker.com/  Docker is a tool for building and running containers that isolate applications from other applications running on the same node.  For detailed information about Docker, see the "Docker User Guide.":https://docs.docker.com/userguide/
+This page describes how to set up the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a workflow step will be run in using "Docker.":https://www.docker.com/  Docker is a tool for building and running containers that isolate applications from other applications running on the same node.  For detailed information about Docker, see the "Docker User Guide.":https://docs.docker.com/userguide/
 
-This page will demonstrate how to:
+This page describes:
 
-# Fetch the arvados/jobs Docker image
-# Manually install additional software into the container
-# Create a new custom image
-# Upload that image to Arvados for use by Crunch jobs
-# Share your image with others
+# "Create a custom image using a Dockerfile":#create
+# "Uploading an image to Arvados":#upload
+# "Sources of pre-built bioinformatics Docker images":#sources
 
 {% include 'tutorial_expectations_workstation' %}
 
 You also need ensure that "Docker is installed,":https://docs.docker.com/installation/ the Docker daemon is running, and you have permission to access Docker.  You can test this by running @docker version@.  If you receive a permission denied error, your user account may need to be added to the @docker@ group.  If you have root access, you can add yourself to the @docker@ group using @$ sudo addgroup $USER docker@ then log out and log back in again; otherwise consult your local sysadmin.
 
-h2. Fetch a starting image
+h2(#create). Create a custom image using a Dockerfile
 
-The easiest way to begin is to start from the "arvados/jobs" image which already has the Arvados SDK installed along with other configuration required for use with Crunch.
+This example shows how to create a Docker image and add the R package.
 
-Download the latest "arvados/jobs" image from the Docker registry:
+First, create new directory called @docker-example@, in that directory create a file called @Dockerfile@.
 
 
-
$ docker pull arvados/jobs:latest
-Pulling repository arvados/jobs
-3132168f2acb: Download complete
-a42b7f2c59b6: Download complete
-e5afdf26a7ae: Download complete
-5cae48636278: Download complete
-7a4f91b70558: Download complete
-a04a275c1fd6: Download complete
-c433ff206a22: Download complete
-b2e539b45f96: Download complete
-073b2581c6be: Download complete
-593915af19dc: Download complete
-32260b35005e: Download complete
-6e5b860c1cde: Download complete
-95f0bfb43d4d: Download complete
-c7fd77eedb96: Download complete
-0d7685aafd00: Download complete
+
$ mkdir docker-example-r-base
+$ cd docker-example-r-base
 

 
 
-h2. Install new packages
-
-Next, enter the container using @docker run@, providing the arvados/jobs image and the program you want to run (in this case the bash shell).
-
 
-
$ docker run --interactive --tty --user root arvados/jobs /bin/bash
-root@fbf1d0f529d5:/#
+
FROM ubuntu:bionic
+RUN apt-get update && apt-get -yq --no-install-recommends install r-base-core
 

 
 
-Next, update the package list using @apt-get update@.
+The "RUN" command is executed inside the container and can be any shell command line.  You are not limited to installing Debian packages.  You may compile programs or libraries from source and install them, edit systemwide configuration files, use other package managers such as @pip@ or @gem@, and perform any other customization necessary to run your program.
 
-
-
root@fbf1d0f529d5:/# apt-get update
-Get:2 http://apt.arvados.org stretch-dev InRelease [3260 B]
-Get:1 http://security-cdn.debian.org/debian-security stretch/updates InRelease [94.3 kB]
-Ign:3 http://cdn-fastly.deb.debian.org/debian stretch InRelease
-Get:4 http://cdn-fastly.deb.debian.org/debian stretch-updates InRelease [91.0 kB]
-Get:5 http://apt.arvados.org stretch-dev/main amd64 Packages [208 kB]
-Get:6 http://cdn-fastly.deb.debian.org/debian stretch Release [118 kB]
-Get:7 http://security-cdn.debian.org/debian-security stretch/updates/main amd64 Packages [499 kB]
-Get:8 http://cdn-fastly.deb.debian.org/debian stretch Release.gpg [2434 B]
-Get:9 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages.diff/Index [10.6 kB]
-Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
-Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
-Fetched 1026 kB in 0s (1384 kB/s)
-Reading package lists... Done
-

-
+You can also visit the "Docker tutorial":https://docs.docker.com/get-started/part2/ for more information and examples.
+
+You should add your Dockerfiles to the same source control repository as the Workflows that use them.
 
-In this example, we will install the "R" statistical language Debian package "r-base-core".  Use @apt-get install@:
+h3. Create a new image
+
+We're now ready to create a new Docker image.  Use @docker build@ to create a new image from the Dockerfile.
 
 
-
root@fbf1d0f529d5:/# apt-get install r-base-core
-Reading package lists... Done
-Building dependency tree
-Reading state information... Done
-The following additional packages will be installed:
-[...]
-done.
+
docker-example-r-base$ docker build -t docker-example-r-base .
 

 
 
+h3. Verify image
+
 Now we can verify that "R" is installed:
 
 
-
root@fbf1d0f529d5:/# R
+
$ docker run -ti docker-example-r-base
+root@57ec8f8b2663:/# R
 
-R version 3.3.3 (2017-03-06) -- "Another Canoe"
-Copyright (C) 2017 The R Foundation for Statistical Computing
+R version 3.4.4 (2018-03-15) -- "Someone to Lean On"
+Copyright (C) 2018 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
-
-R is free software and comes with ABSOLUTELY NO WARRANTY.
-You are welcome to redistribute it under certain conditions.
-Type 'license()' or 'licence()' for distribution details.
-
-R is a collaborative project with many contributors.
-Type 'contributors()' for more information and
-'citation()' on how to cite R or R packages in publications.
-
-Type 'demo()' for some demos, 'help()' for on-line help, or
-'help.start()' for an HTML browser interface to help.
-Type 'q()' to quit R.
-
->
 

 
 
-Note that you are not limited to installing Debian packages.  You may compile programs or libraries from source and install them, edit systemwide configuration files, use other package managers such as @pip@ or @gem@, and perform any other customization necessary to run your program.
+h2(#upload). Upload your image
 
-h2. Create a new image
-
-We're now ready to create a new Docker image.  First, quit the container, then use @docker commit@ to create a new image from the stopped container.  The container id can be found in the default hostname of the container displayed in the prompt, in this case @fbf1d0f529d5@:
+Finally, we are ready to upload the new Docker image to Arvados.  Use @arv-keepdocker@ with the image repository name to upload the image.  Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
 
 
-
root@fbf1d0f529d5:/# exit
-$ docker commit fbf1d0f529d5 arvados/jobs-with-r
-sha256:2818853ff9f9af5d7f77979803baac9c4710790ad2b84c1a754b02728fdff205
-$ docker images
-$ docker images |head
-REPOSITORY            TAG                 IMAGE ID            CREATED             SIZE
-arvados/jobs-with-r   latest              2818853ff9f9        9 seconds ago       703.1 MB
-arvados/jobs          latest              12b9f859d48c        4 days ago          362 MB
-

-
-
-h2. Upload your image
+
$ arv-keepdocker docker-example-r-base
+2020-06-29 13:48:19 arvados.arv_put[769] INFO: Creating new cache file at /home/peter/.cache/arvados/arv-put/39ddb51ebf6c5fcb3d713b5969466967
+206M / 206M 100.0% 2020-06-29 13:48:21 arvados.arv_put[769] INFO:
 
-Finally, we are ready to upload the new Docker image to Arvados.  Use @arv-keepdocker@ with the image repository name to upload the image.  Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
+2020-06-29 13:48:21 arvados.arv_put[769] INFO: Collection saved as 'Docker image docker-example-r-base:latest sha256:edd10'
+zzzzz-4zz18-0tayximqcyb6uf8
 
-
-
$ arv-keepdocker arvados/jobs-with-r
-703M / 703M 100.0%
-Collection saved as 'Docker image arvados/jobs-with-r:latest 2818853ff9f9'
-qr1hi-4zz18-abcdefghijklmno
-$ arv-keepdocker
+$ arv-keepdocker images
 REPOSITORY                      TAG         IMAGE ID      COLLECTION                     CREATED
-arvados/jobs-with-r             latest      2818853ff9f9  qr1hi-4zz18-abcdefghijklmno    Tue Jan 17 20:35:53 2017
+docker-example-r-base           latest      sha256:edd10  zzzzz-4zz18-0tayximqcyb6uf8    Mon Jun 29 17:46:16 2020
 

 
 
@@ -156,14 +91,24 @@ You are now able to specify the runtime environment for your program using @Dock
 
 hints:
   DockerRequirement:
-    dockerPull: arvados/jobs-with-r
+    dockerPull: docker-example-r-base
 

 
-h2. Share Docker images
+h3. Uploading Docker images to a shared project
 
-Docker images are subject to normal Arvados permissions.  If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv-keepdocker@ with the @--project-uuid@ option to upload the image to a shared project.
+Docker images are subject to normal Arvados permissions.  If wish to share your Docker image with others you should use @arv-keepdocker@ with the @--project-uuid@ option to add the image to a shared project and ensure that metadata is set correctly.
 
 
-
$ arv-keepdocker arvados/jobs-with-r --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx
+
$ arv-keepdocker docker-example-r-base --project-uuid zzzzz-j7d0g-xxxxxxxxxxxxxxx
 

 
+
+h2(#sources). Sources of pre-built images
+
+In addition to creating your own contianers, there are a number of resources where you can find bioinformatics tools already wrapped in container images:
+
+"BioContainers":https://biocontainers.pro/
+
+"Dockstore":https://dockstore.org/
+
+"Docker Hub":https://hub.docker.com/
diff --git a/doc/user/topics/keep.html.textile.liquid b/doc/user/topics/keep.html.textile.liquid
index 68b6a87d09..c415cebbdd 100644
--- a/doc/user/topics/keep.html.textile.liquid
+++ b/doc/user/topics/keep.html.textile.liquid
@@ -38,7 +38,7 @@ notextile. 
~$ cd /scratch/you
 
 When you run this command, you may get this API warning:
 
-notextile. 
WARNING:root:API lookup failed for collection 204e43b8a1185621ca55a94839582e6f+67108864 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/204e43b8a1185621ca55a94839582e6f%2B67108864?alt=json returned "Not Found">)

+notextile. 
WARNING:root:API lookup failed for collection 204e43b8a1185621ca55a94839582e6f+67108864 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://zzzzz.arvadosapi.com/arvados/v1/collections/204e43b8a1185621ca55a94839582e6f%2B67108864?alt=json returned "Not Found">)

 
 This happens because @arv-get@ tries to find a collection with this identifier.  When that fails, it emits this warning, then looks for a datablock instead, which succeeds.
 
diff --git a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
deleted file mode 100644
index 544ccbd35e..0000000000
--- a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
+++ /dev/null
@@ -1,173 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "Using GATK with Arvados"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This tutorial demonstrates how to use the Genome Analysis Toolkit (GATK) with Arvados. In this example we will install GATK and then create a VariantFiltration job to assign pass/fail scores to variants in a VCF file.
-
-{% include 'tutorial_expectations' %}
-
-h2. Installing GATK
-
-Download the GATK binary tarball[1] -- e.g., @GenomeAnalysisTK-2.6-4.tar.bz2@ -- and "copy it to your Arvados VM":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-
-
-
~$ arv-put GenomeAnalysisTK-2.6-4.tar.bz2
-c905c8d8443a9c44274d98b7c6cfaa32+94
-

-
-
-Next, you need the GATK Resource Bundle[2].  This may already be available in Arvados.  If not, you will need to download the files listed below and put them into Keep.
-
-
-
~$ arv keep ls -s d237a90bae3870b3b033aea1e99de4a9+10820
-  50342 1000G_omni2.5.b37.vcf.gz
-      1 1000G_omni2.5.b37.vcf.gz.md5
-    464 1000G_omni2.5.b37.vcf.idx.gz
-      1 1000G_omni2.5.b37.vcf.idx.gz.md5
-  43981 1000G_phase1.indels.b37.vcf.gz
-      1 1000G_phase1.indels.b37.vcf.gz.md5
-    326 1000G_phase1.indels.b37.vcf.idx.gz
-      1 1000G_phase1.indels.b37.vcf.idx.gz.md5
- 537210 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.gz
-      1 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.gz.md5
-   3473 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.idx.gz
-      1 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.idx.gz.md5
-  19403 Mills_and_1000G_gold_standard.indels.b37.vcf.gz
-      1 Mills_and_1000G_gold_standard.indels.b37.vcf.gz.md5
-    536 Mills_and_1000G_gold_standard.indels.b37.vcf.idx.gz
-      1 Mills_and_1000G_gold_standard.indels.b37.vcf.idx.gz.md5
-  29291 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.gz.md5
-    565 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.idx.gz.md5
-  37930 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.gz.md5
-    592 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.idx.gz.md5
-5898484 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam
-    112 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.bai.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.bai.gz.md5
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.md5
-   3837 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.gz.md5
-     65 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.idx.gz.md5
- 275757 dbsnp_137.b37.excluding_sites_after_129.vcf.gz
-      1 dbsnp_137.b37.excluding_sites_after_129.vcf.gz.md5
-   3735 dbsnp_137.b37.excluding_sites_after_129.vcf.idx.gz
-      1 dbsnp_137.b37.excluding_sites_after_129.vcf.idx.gz.md5
- 998153 dbsnp_137.b37.vcf.gz
-      1 dbsnp_137.b37.vcf.gz.md5
-   3890 dbsnp_137.b37.vcf.idx.gz
-      1 dbsnp_137.b37.vcf.idx.gz.md5
-  58418 hapmap_3.3.b37.vcf.gz
-      1 hapmap_3.3.b37.vcf.gz.md5
-    999 hapmap_3.3.b37.vcf.idx.gz
-      1 hapmap_3.3.b37.vcf.idx.gz.md5
-      3 human_g1k_v37.dict.gz
-      1 human_g1k_v37.dict.gz.md5
-      2 human_g1k_v37.fasta.fai.gz
-      1 human_g1k_v37.fasta.fai.gz.md5
- 849537 human_g1k_v37.fasta.gz
-      1 human_g1k_v37.fasta.gz.md5
-      1 human_g1k_v37.stats.gz
-      1 human_g1k_v37.stats.gz.md5
-      3 human_g1k_v37_decoy.dict.gz
-      1 human_g1k_v37_decoy.dict.gz.md5
-      2 human_g1k_v37_decoy.fasta.fai.gz
-      1 human_g1k_v37_decoy.fasta.fai.gz.md5
- 858592 human_g1k_v37_decoy.fasta.gz
-      1 human_g1k_v37_decoy.fasta.gz.md5
-      1 human_g1k_v37_decoy.stats.gz
-      1 human_g1k_v37_decoy.stats.gz.md5
-

-
-
-h2. Submit a GATK job
-
-The Arvados distribution includes an example crunch script ("crunch_scripts/GATK2-VariantFiltration":https://dev.arvados.org/projects/arvados/repository/revisions/master/entry/crunch_scripts/GATK2-VariantFiltration) that runs the GATK VariantFiltration tool with some default settings.
-
-
-
~$ src_version=76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-~$ vcf_input=5ee633fe2569d2a42dd81b07490d5d13+82
-~$ gatk_binary=c905c8d8443a9c44274d98b7c6cfaa32+94
-~$ gatk_bundle=d237a90bae3870b3b033aea1e99de4a9+10820
-~$ cat >the_job <<EOF
-{
- "script":"GATK2-VariantFiltration",
- "repository":"arvados",
- "script_version":"$src_version",
- "script_parameters":
- {
-  "input":"$vcf_input",
-  "gatk_binary_tarball":"$gatk_binary",
-  "gatk_bundle":"$gatk_bundle"
- }
-}
-EOF
-

-
-
-* @"input"@ is collection containing the source VCF data. Here we are using an exome report from PGP participant hu34D5B9.
-* @"gatk_binary_tarball"@ is a Keep collection containing the GATK 2 binary distribution[1] tar file.
-* @"gatk_bundle"@ is a Keep collection containing the GATK resource bundle[2].
-
-Now start a job:
-
-
-
~$ arv job create --job "$(cat the_job)"
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-n9k7qyp7bs5b9d4",
- "kind":"arvados#job",
- "etag":"9j99n1feoxw3az448f8ises12",
- "uuid":"qr1hi-8i9sb-n9k7qyp7bs5b9d4",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-17T19:02:15Z",
- "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-17T19:02:15Z",
- "updated_at":"2013-12-17T19:02:15Z",
- "submit_id":null,
- "priority":null,
- "script":"GATK2-VariantFiltration",
- "script_parameters":{
-  "input":"5ee633fe2569d2a42dd81b07490d5d13+82",
-  "gatk_binary_tarball":"c905c8d8443a9c44274d98b7c6cfaa32+94",
-  "gatk_bundle":"d237a90bae3870b3b033aea1e99de4a9+10820"
- },
- "script_version":"76588bfc57f33ea1b36b82ca7187f465b73b4ca4",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":null,
- "finished_at":null,
- "output":null,
- "success":null,
- "running":null,
- "is_locked_by_uuid":null,
- "log":null,
- "runtime_constraints":{},
- "tasks_summary":{}
-}
-

-
-
-Once the job completes, the output can be found in hu34D5B9-exome-filtered.vcf:
-
-
~$ arv keep ls bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-hu34D5B9-exome-filtered.vcf
-

-
-
-h2. Notes
-
-fn1. "Download the GATK tools":http://www.broadinstitute.org/gatk/download
-
-fn2. "Information about the GATK resource bundle":http://gatkforums.broadinstitute.org/discussion/1213/whats-in-the-resource-bundle-and-how-can-i-get-it and "direct download link":ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/2.5/b37/ (if prompted, submit an empty password)
diff --git a/doc/user/topics/tutorial-job1.html.textile.liquid b/doc/user/topics/tutorial-job1.html.textile.liquid
deleted file mode 100644
index f7a2060101..0000000000
--- a/doc/user/topics/tutorial-job1.html.textile.liquid
+++ /dev/null
@@ -1,214 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "Running a Crunch job on the command line"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This tutorial introduces how to run individual Crunch jobs using the @arv@ command line tool.
-
-{% include 'tutorial_expectations' %}
-
-You will create a job to run the "hash" Crunch script.  The "hash" script computes the MD5 hash of each file in a collection.
-
-h2. Jobs
-
-Crunch pipelines consist of one or more jobs.  A "job" is a single run of a specific version of a Crunch script with a specific input.  You can also run jobs individually.
-
-A request to run a Crunch job are is described using a JSON object.  For example:
-
-
-
~$ cat >~/the_job <<EOF
-{
- "script": "hash",
- "repository": "arvados",
- "script_version": "master",
- "script_parameters": {
-  "input": "c1bad4b39ca5a924e481008009d94e32+210"
- },
- "no_reuse": "true"
-}
-EOF
-

-
-
-* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
-* @<~/the_job@ redirects standard output to a file called @~/the_job@.
-* @"repository"@ is the name of a Git repository to search for the script version.  You can access a list of available git repositories on the Arvados Workbench under "*Code repositories*":{{site.arvados_workbench_host}}/repositories.
-* @"script_version"@ specifies the version of the script that you wish to run.  This can be in the form of an explicit Git revision hash, a tag, or a branch.  Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
-* @"script"@ specifies the name of the script to run.  The script must be given relative to the @crunch_scripts/@ subdirectory of the Git repository.
-* @"script_parameters"@ are provided to the script.  In this case, the input is the PGP data Collection that we "put in Keep earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-* Setting the @"no_reuse"@ flag tells Crunch not to reuse work from past jobs.  This helps ensure that you can watch a new Job process for the rest of this tutorial, without reusing output from a past run that you made, or somebody else marked as public.  (If you want to experiment, after the first run below finishes, feel free to edit this job to remove the @"no_reuse"@ line and resubmit it.  See what happens!)
-
-Use @arv job create@ to actually submit the job.  It should print out a JSON object which describes the newly created job:
-
-
-
~$ arv job create --job "$(cat ~/the_job)"
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"ax3cn7w9whq2hdh983yxvq09p",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:32Z",
- "updated_at":"2013-12-16T20:44:33Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
-  "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":null,
- "finished_at":null,
- "output":null,
- "success":null,
- "running":null,
- "is_locked_by_uuid":null,
- "log":null,
- "runtime_constraints":{},
- "tasks_summary":{}
-}
-

-
-
-The job is now queued and will start running as soon as it reaches the front of the queue.  Fields to pay attention to include:
-
- * @"uuid"@ is the unique identifier for this specific job.
- * @"script_version"@ is the actual revision of the script used.  This is useful if the version was described using the "repository:branch" format.
-
-h2. Monitor job progress
-
-Go to "*Recent jobs*":{{site.arvados_workbench_host}}/jobs in Workbench.  Your job should be near the top of the table.  This table refreshes automatically.  When the job has completed successfully, it will show finished in the *Status* column.
-
-h2. Inspect the job output
-
-On the "Workbench Dashboard":{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table.  Click on the link under *Output* for your job to go to the files page with the job output.  The files page lists all the files that were output by the job.  Click on the link under the *file* column to view a file, or click on the download button  to download the output file.
-
-On the command line, you can use @arv job get@ to access a JSON object describing the output:
-
-
-
~$ arv job get --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"1bk98tdj0qipjy0rvrj03ta5r",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":null,
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:55Z",
- "updated_at":"2013-12-16T20:44:55Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
-  "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":"2013-12-16T20:44:36Z",
- "finished_at":"2013-12-16T20:44:53Z",
- "output":"dd755dbc8d49a67f4fe7dc843e4f10a6+54",
- "success":true,
- "running":false,
- "is_locked_by_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "log":"2afdc6c8b67372ffd22d8ce89d35411f+91",
- "runtime_constraints":{},
- "tasks_summary":{
-  "done":2,
-  "running":0,
-  "failed":0,
-  "todo":0
- }
-}
-

-
-
-* @"output"@ is the unique identifier for this specific job's output.  This is a Keep collection.  Because the output of Arvados jobs should be deterministic, the known expected output is dd755dbc8d49a67f4fe7dc843e4f10a6+54.
-
-Now you can list the files in the collection:
-
-
-
~$ arv keep ls dd755dbc8d49a67f4fe7dc843e4f10a6+54
-./md5sum.txt
-

-
-
-This collection consists of the @md5sum.txt@ file.  Use @arv-get@ to show the contents of the @md5sum.txt@ file:
-
-
-
~$ arv-get dd755dbc8d49a67f4fe7dc843e4f10a6+54/md5sum.txt
-44b8ae3fde7a8a88d2f7ebd237625b4f ./var-GS000016015-ASM.tsv.bz2
-

-
-
-This MD5 hash matches the MD5 hash which we "computed earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-
-h2. The job log
-
-When the job completes, you can access the job log.  On the Workbench, visit "*Recent jobs*":{{site.arvados_workbench_host}}/jobs %(rarr)→% your job's UUID under the *uuid* column %(rarr)→% the collection link on the *log* row.
-
-On the command line, the Keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection.  You can list the files in the collection:
-
-
-
~$ arv keep ls xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91
-./qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
-

-
-
-The log collection consists of one log file named with the job's UUID.  You can access it using @arv-get@:
-
-
-
~$ arv-get xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91/qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
-2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  check slurm allocation
-2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  node compute13 - 8 slots
-2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start
-2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Clean-work-dir exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Install exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script hash
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  runtime_constraints {"max_tasks_per_node":0}
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start level 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 0 done, 0 running, 1 todo
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 started on compute13.1
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 0 done, 1 running, 0 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 success in 1 seconds
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 output
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  wait for last 0 children to finish
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start level 1
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 started on compute13.1
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 1 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 success in 13 seconds
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 output dd755dbc8d49a67f4fe7dc843e4f10a6+54
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  wait for last 0 children to finish
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 2 done, 0 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  release job allocation
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Freeze not implemented
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  collate
-2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  output dd755dbc8d49a67f4fe7dc843e4f10a6+54+K@qr1hi
-2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  finish
-

-
diff --git a/doc/user/tutorials/add-new-repository.html.textile.liquid b/doc/user/tutorials/add-new-repository.html.textile.liquid
index 9d8e768a78..e28b961238 100644
--- a/doc/user/tutorials/add-new-repository.html.textile.liquid
+++ b/doc/user/tutorials/add-new-repository.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-Arvados repositories are managed through the Git revision control system. You can use these repositories to store your crunch scripts and run them in the arvados cluster.
+Arvados supports managing git repositories. You can access these repositories using your Arvados credentials and share them with other Arvados users.
 
 {% include 'tutorial_expectations' %}
 
diff --git a/doc/user/tutorials/git-arvados-guide.html.textile.liquid b/doc/user/tutorials/git-arvados-guide.html.textile.liquid
index 2e255219d2..ad719a66e4 100644
--- a/doc/user/tutorials/git-arvados-guide.html.textile.liquid
+++ b/doc/user/tutorials/git-arvados-guide.html.textile.liquid
@@ -9,20 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This tutorial describes how to work with a new Arvados git repository. Working with an Arvados git repository is analogous to working with other public git repositories. It will show you how to upload custom scripts to a remote Arvados repository, so you can use it in Arvados pipelines.
+This tutorial describes how to work with an Arvados-managed git repository. Working with an Arvados git repository is very similar to working with other public git repositories.
 
 {% include 'tutorial_expectations' %}
 
 {% include 'tutorial_git_repo_expectations' %}
 
-{% include 'notebox_begin' %}
-For more information about using Git, try
-
-
$ man gittutorial

- or *"search Google for Git tutorials":http://google.com/#q=git+tutorial*.
-{% include 'notebox_end' %}
-
-h2. Cloning an Arvados repository
+h2. Cloning a git repository
 
 Before you start using Git, you should do some basic configuration (you only need to do this the first time):
 
@@ -65,33 +58,22 @@ Create a git branch named *tutorial_branch* in the *tutorial* Arvados git reposi
 
 h2. Adding scripts to an Arvados repository
 
-Arvados crunch scripts need to be added in a *crunch_scripts* subdirectory in the repository. If this subdirectory does not exist, first create it in the local repository and change to that directory:
-
-
-
~/tutorial$ mkdir crunch_scripts
-~/tutorial$ cd crunch_scripts

-
-
-Next, using @nano@ or your favorite Unix text editor, create a new file called @hash.py@ in the @crunch_scripts@ directory.
-
-notextile. 
~/tutorial/crunch_scripts$ nano hash.py

-
-Add the following code to compute the MD5 hash of each file in a collection
+A git repository is a good place to store the CWL workflows that you run on Arvados.
 
- {% code 'tutorial_hash_script_py' as python %} 
+First, create a simple CWL CommandLineTool:
 
-Make the file executable:
+notextile. 
~/tutorials$ nano hello.cwl

 
-notextile. 
~/tutorial/crunch_scripts$ chmod +x hash.py

+ {% code 'tutorial_hello_cwl' as yaml %} 
 
 Next, add the file to the git repository.  This tells @git@ that the file should be included on the next commit.
 
-notextile. 
~/tutorial/crunch_scripts$ git add hash.py

+notextile. 
~/tutorial$ git add hello.cwl

 
 Next, commit your changes.  All staged changes are recorded into the local git repository:
 
 
-
~/tutorial/crunch_scripts$ git commit -m "my first script"
+
~/tutorial$ git commit -m "my first script"
 

 
 
@@ -102,4 +84,4 @@ Finally, upload your changes to the remote repository:
 

 
 
-Although this tutorial shows how to add a python script to Arvados, the same steps can be used to add any of your custom bash, R, or python scripts to an Arvados repository.
+The same steps can be used to add any of your custom bash, R, or python scripts to an Arvados repository.
diff --git a/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid b/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
index 2375e8b3d5..c4b9e31ece 100644
--- a/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
@@ -11,45 +11,42 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 During it's lifetime, a keep collection can be in various states. These states are *persisted*, *expiring*, *trashed*  and *permanently deleted*.
 
+The nominal state is *persisted* which means the data can be can be accessed normally and will be retained indefinitely.
+
 A collection is *expiring* when it has a *trash_at* time in the future. An expiring collection can be accessed as normal, but is scheduled to be trashed automatically at the *trash_at* time.
 
 A collection is *trashed* when it has a *trash_at* time in the past. The *is_trashed* attribute will also be "true". The delete operation immediately puts the collection in the trash by setting the *trash_at* time to "now". Once trashed, the collection is no longer readable through normal data access APIs. The collection will have *delete_at* set to some time in the future. The trashed collection is recoverable until the delete_at time passes, at which point the collection is permanently deleted.
 
-# "*Collection lifecycle attributes*":#collection_attributes
 # "*Deleting / trashing collections*":#delete-collection
 # "*Recovering trashed collections*":#trash-recovery
+# "*Collection lifecycle attributes*":#collection_attributes
 
 {% include 'tutorial_expectations' %}
 
-h2(#collection_attributes). Collection lifecycle attributes
-
-As listed above the attributes that are used to manage a collection lifecycle are it's *is_trashed*, *trash_at*, and *delete_at*. The table below lists the values of these attributes and how they influence the state of a collection and it's accessibility.
-
-table(table table-bordered table-condensed).
-|_. collection state|_. is_trashed|_. trash_at|_. delete_at|_. get|_. list|_. list?include_trash=true|_. can be modified|
-|persisted collection|false |null |null |yes |yes |yes |yes |
-|expiring collection|false |future |future |yes  |yes |yes |yes |
-|trashed collection|true |past |future |no |no |yes |only is_trashed, trash_at and delete_at attribtues|
-|deleted collection|true|past |past |no |no |no |no |
-
 h2(#delete-collection). Deleting / trashing collections
 
 A collection can be deleted using either the arv command line tool or the workbench.
 
+h3. Trashing a collection using workbench
+
+To trash a collection using workbench, go to the Data collections tab in the project, and use the  trash icon for this collection row.
+
 h3. Trashing a collection using arv command line tool
 
 
-arv collection delete --uuid=qr1hi-4zz18-xxxxxxxxxxxxxxx
+arv collection delete --uuid=zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
-h3. Trashing a collection using workbench
-
-To trash a collection using workbench, go to the Data collections tab in the project, and use the trash icon for this collection row.
-
 h2(#trash-recovery). Recovering trashed collections
 
 A collection can be un-trashed / recovered using either the arv command line tool or the workbench.
 
+h3. Un-trashing a collection using workbench
+
+To untrash a collection using workbench, go to trash page on workbench by clicking on the "Trash" icon in the top navigation in workbench and use the recycle icon or selection dropdown option.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/trash-button-topnav.png!
+
 h3. Un-trashing a collection using arv command line tool
 
 You can list the trashed collections using the list command.
@@ -61,11 +58,16 @@ arv collection list --include-trash=true --filters '[["is_trashed", "=", "true"]
 You can then untrash a particular collection using arv using it's uuid.
 
 
-arv collection untrash --uuid=qr1hi-4zz18-xxxxxxxxxxxxxxx
+arv collection untrash --uuid=zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
-h3. Un-trashing a collection using workbench
+h2(#collection_attributes). Collection lifecycle attributes
 
-To untrash a collection using workbench, go to trash page on workbench by clicking on the "Trash" icon in the top navigation in workbench and use the recycle icon or selection dropdown option.
+As listed above the attributes that are used to manage a collection lifecycle are it's *is_trashed*, *trash_at*, and *delete_at*. The table below lists the values of these attributes and how they influence the state of a collection and it's accessibility.
 
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/trash-button-topnav.png!
+table(table table-bordered table-condensed).
+|_. collection state|_. is_trashed|_. trash_at|_. delete_at|_. get|_. list|_. list?include_trash=true|_. can be modified|
+|persisted collection|false |null |null |yes |yes |yes |yes |
+|expiring collection|false |future |future |yes  |yes |yes |yes |
+|trashed collection|true |past |future |no |no |yes |only is_trashed, trash_at and delete_at attribtues|
+|deleted collection|true|past |past |no |no |no |no |
diff --git a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
index f206d302de..05924f8475 100644
--- a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
@@ -11,11 +11,39 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Arvados Data collections can be downloaded using either the arv commands or using Workbench.
 
-# "*Downloading using arv commands*":#download-using-arv
-# "*Downloading using Workbench*":#download-using-workbench
-# "*Downloading a shared collection using Workbench*":#download-shared-collection
+# "*Download using Workbench*":#download-using-workbench
+# "*Sharing collections*":#download-shared-collection
+# "*Download using command line tools*":#download-using-arv
 
-h2(#download-using-arv). Downloading using arv commands
+h2(#download-using-workbench). Download using Workbench
+
+You can also download Arvados data collections using the Workbench.
+
+Visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu, select your *Home* project. You will see the *Data collections* tab, which lists the collections in this project.
+
+You can access the contents of a collection by clicking on the * Show* button next to the collection. This will take you to the collection's page. Using this page you can see the collection's contents, and download individual files.
+
+You can now download the collection files by clicking on the  button(s).
+
+h2(#download-shared-collection). Sharing collections
+
+h3. Sharing with other Arvados users
+
+Collections can be shared with other users on the Arvados cluster by sharing the parent project.  Navigate to the parent project using the "breadcrumbs" bar, then click on the *Sharing* tab.  From the sharing tab, you can choose which users or groups to share with, and their level of access.
+
+h3. Creating a special download URL
+
+To share a collection with users that do not have an account on your Arvados cluster, visit the collection page using Workbench as described in the above section. Once on this page, click on the Create sharing link button.
+
+This will create a sharing link for the collection as shown below. You can copy the sharing link in this page and share it with other users.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/shared-collection.png!
+
+A user with this url can download this collection by simply accessing this url using browser. It will present a downloadable version of the collection as shown below.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/download-shared-collection.png!
+
+h2(#download-using-arv). Download using command line tools
 
 {% include 'tutorial_expectations' %}
 
@@ -24,38 +52,35 @@ You can download Arvados data collections using the command line tools @arv-ls@
 Use @arv-ls@ to view the contents of a collection:
 
 
-
~$ arv-ls c1bad4b39ca5a924e481008009d94e32+210
-var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls ae480c5099b81e17267b7445e35b4bc7+180
+./HWI-ST1027_129_D0THKACXX.1_1.fastq
+./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
-
~$ arv-ls 887cd41e9c613463eab2f0d885c6dd96+83
-alice.txt
-bob.txt
-carol.txt
-

-
-
-Use @-s@ to print file sizes rounded up to the nearest kilobyte:
+Use @-s@ to print file sizes, in kilobytes, rounded up:
 
 
-
~$ arv-ls -s c1bad4b39ca5a924e481008009d94e32+210
-221887 var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls -s ae480c5099b81e17267b7445e35b4bc7+180
+     12258 ./HWI-ST1027_129_D0THKACXX.1_1.fastq
+     12258 ./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 Use @arv-get@ to download the contents of a collection and place it in the directory specified in the second argument (in this example, @.@ for the current directory):
 
 
-
~$ arv-get c1bad4b39ca5a924e481008009d94e32+210/ .
-~$ ls var-GS000016015-ASM.tsv.bz2
-var-GS000016015-ASM.tsv.bz2
+
~$ $ arv-get ae480c5099b81e17267b7445e35b4bc7+180/ .
+23 MiB / 23 MiB 100.0%
+~$ ls
+HWI-ST1027_129_D0THKACXX.1_1.fastq  HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 You can also download individual files:
 
 
-
~$ arv-get 887cd41e9c613463eab2f0d885c6dd96+83/alice.txt .
+
~$ arv-get ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_1.fastq .
+11 MiB / 11 MiB 100.0%
 

 
 
@@ -65,33 +90,9 @@ If your cluster is "configured to be part of a federation":{{site.baseurl}}/admi
 
 If you request a collection by portable data hash, it will first search the home cluster, then search federated clusters.
 
-You may also request a collection by UUID.  In this case, it will contact the cluster named in the UUID prefix (in this example, @qr1hi@).
+You may also request a collection by UUID.  In this case, it will contact the cluster named in the UUID prefix (in this example, @zzzzz@).
 
 
-
~$ arv-get qr1hi-4zz18-fw6dnjxtkvzdewt/ .
+
~$ arv-get zzzzz-4zz18-fw6dnjxtkvzdewt/ .
 

 
-
-h2(#download-using-workbench). Downloading using Workbench
-
-You can also download Arvados data collections using the Workbench.
-
-Visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu, select your *Home* project. You will see the *Data collections* tab, which lists the collections in this project.
-
-You can access the contents of a collection by clicking on the * Show* button next to the collection. This will take you to the collection's page. Using this page you can see the collection's contents, download individual files, and set sharing options.
-
-You can now download the collection files by clicking on the  button(s).
-
-h2(#download-shared-collection). Downloading a shared collection using Workbench
-
-Collections can be shared to allow downloads by anonymous users.
-
-To share a collection with anonymous users, visit the collection page using Workbench as described in the above section. Once on this page, click on the Create sharing link button.
-
-This will create a sharing link for the collection as shown below. You can copy the sharing link in this page and share it with other users.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/shared-collection.png!
-
-A user with this url can download this collection by simply accessing this url using browser. It will present a downloadable version of the collection as shown below.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/download-shared-collection.png!
diff --git a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
index e176021992..060ae2acbe 100644
--- a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from GNU/Linux"
+title: "Access Keep as a GNU/Linux filesystem"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,17 +9,16 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This tutoral describes how to access Arvados collections on GNU/Linux using traditional filesystem tools by mounting Keep as a file system using @arv-mount@.
+GNU/Linux users can use @arv-mount@ or Gnome to mount Keep as a file system in order to access Arvados collections using traditional filesystem tools.
 
 {% include 'tutorial_expectations' %}
 
-h2. Arv-mount
+# "*Mounting at the command line with arv-mount*":#arv-mount
+# "*Mounting in Gnome File manager*":#gnome
 
-@arv-mount@ provides several features:
+h2(#arv-mount). Arv-mount
 
-* You can browse, open and read Keep entries as if they are regular files.
-* It is easy for existing tools to access files in Keep.
-* Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
+@arv-mount@ provides a file system view of Arvados Keep using File System in Userspace (FUSE).  You can browse, open and read Keep entries as if they are regular files, and existing tools can access files in Keep.  Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
 
 The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory.  To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by UUID or portable data hash. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
 
@@ -59,3 +58,11 @@ Not supported:
 If multiple clients (separate instances of arv-mount or other arvados applications) modify the same file in the same collection within a short time interval, this may result in a conflict.  In this case, the most recent commit wins, and the "loser" will be renamed to a conflict file in the form @name~YYYYMMDD-HHMMSS~conflict~@.
 
 Please note this feature is in beta testing.  In particular, the conflict mechanism is itself currently subject to race conditions with potential for data loss when a collection is being modified simultaneously by multiple clients.  This issue will be resolved in future development.
+
+h2(#gnome). Mounting in Gnome File manager
+
+As an alternative to @arv-mount@ you can also access the WebDAV mount through the Gnome File manager.
+
+# Open "Files"
+# On the left sidebar, click on "Other Locations"
+# At the bottom of the window, enter @davs://collections.ClusterID.example.com/@  When prompted for credentials, enter username "arvados" and a valid Arvados token in the @Password@ field.
diff --git a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
index 9397d61e05..911b8808eb 100644
--- a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from OS X"
+title: "Access Keep from macOS Finder"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,16 +9,16 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-OS X users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
+Users of macOS can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in Finder (read-only)
 
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, enter username "arvados" and paste a valid Arvados token for the @Password@ field.
 
 This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
 
 h3. Accessing a specific collection in Keep (read-write)
 
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/c=your-collection-uuid@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
 
 This collection is now accessible read/write.
diff --git a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
index 29b28fff9c..a40a997ba1 100644
--- a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from Windows"
+title: "Access Keep from Windows File Explorer"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -11,7 +11,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Windows users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in File Explorer (read-only)
 
 Use the 'Map network drive' functionality, and enter @https://collections.ClusterID.example.com/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
 
diff --git a/doc/user/tutorials/tutorial-keep.html.textile.liquid b/doc/user/tutorials/tutorial-keep.html.textile.liquid
index ec7086db96..21efc475c5 100644
--- a/doc/user/tutorials/tutorial-keep.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep.html.textile.liquid
@@ -9,13 +9,44 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-Arvados Data collections can be uploaded using either the @arv-put@ command line tool or using Workbench.
+Arvados Data collections can be uploaded using either Workbench or the @arv-put@ command line tool.
 
-# "*Upload using command line tool*":#upload-using-command
 # "*Upload using Workbench*":#upload-using-workbench
+# "*Creating projects*":#creating-projects
+# "*Upload using command line tool*":#upload-using-command
+
+h2(#upload-using-workbench). Upload using Workbench
+
+To upload using Workbench, visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu and select your *Home* project or any other project of your choosing.  You will see the *Data collections* tab for this project, which lists the collections in this project.
+
+To upload files into a new collection, click on *Add data* dropdown menu and select *Upload files from my computer*.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-using-workbench.png!
+
+
This will create a new empty collection in your chosen project and will take you to the *Upload* tab for that collection.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-tab-in-new-collection.png!
+
+Click on the *Browse...* button and select the files you would like to upload. Selected files will be added to a list of files to be uploaded. After you are done selecting files to upload, click on the * Start* button to start upload. This will start uploading files to Arvados and Workbench will show you the progress bar. When upload is completed, you will see an indication to that effect.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/files-uploaded.png!
+
+*Note:* If you leave the collection page during the upload, the upload process will be aborted and you will need to upload the files again.
+
+*Note:* You can also use the Upload tab to add additional files to an existing collection.
 
 notextile. 

 
+h2(#creating-projects). Creating projects
+
+Files are organized into Collections, and Collections are organized by Projects.
+
+Click on *Projects* → *Add a new project* to add a top level project.
+
+To create a subproject, navigate to the parent project, and click on *Add a subproject*.
+
+See "Sharing collections":tutorial-keep-get.html#download-shared-collection for information about sharing projects and collections with other users.
+
 h2(#upload-using-command). Upload using command line tool
 
 {% include 'tutorial_expectations' %}
@@ -25,12 +56,12 @@ To upload a file to Keep using @arv-put@:
 
~$ arv-put var-GS000016015-ASM.tsv.bz2
 216M / 216M 100.0%
 Collection saved as ...
-qr1hi-4zz18-xxxxxxxxxxxxxxx
+zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
 
 
-The output value @qr1hi-4zz18-xxxxxxxxxxxxxxx@ is the uuid of the Arvados collection created.
+The output value @zzzzz-4zz18-xxxxxxxxxxxxxxx@ is the uuid of the Arvados collection created.
 
 Note: The file used in this example is a freely available TSV file containing variant annotations from the "Personal Genome Project (PGP)":http://www.pgp-hms.org participant "hu599905":https://my.pgp-hms.org/profile/hu599905), downloadable "here":https://warehouse.pgp-hms.org/warehouse/f815ec01d5d2f11cb12874ab2ed50daa+234+K@ant/var-GS000016015-ASM.tsv.bz2. Alternatively, you can replace @var-GS000016015-ASM.tsv.bz2@ with the name of any file you have locally, or you could get the TSV file by "downloading it from Keep.":{{site.baseurl}}/user/tutorials/tutorial-keep-get.html
 
@@ -44,7 +75,7 @@ Note: The file used in this example is a freely available TSV file containing va
 ~$ arv-put tmp
 0M / 0M 100.0%
 Collection saved as ...
-qr1hi-4zz18-yyyyyyyyyyyyyyy
+zzzzz-4zz18-yyyyyyyyyyyyyyy
 

 
 
@@ -63,23 +94,3 @@ To move the collection to a different project, check the box at the left of the
 Click on the * Show* button next to the collection's listing on a project page to go to the Workbench page for your collection.  On this page, you can see the collection's contents, download individual files, and set sharing options.
 
 notextile. 
-
-h2(#upload-using-workbench). Upload using Workbench
-
-To upload using Workbench, visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu and select your *Home* project or any other project of your choosing.  You will see the *Data collections* tab for this project, which lists the collections in this project.
-
-To upload files into a new collection, click on *Add data* dropdown menu and select *Upload files from my computer*.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-using-workbench.png!
-
-
This will create a new empty collection in your chosen project and will take you to the *Upload* tab for that collection.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-tab-in-new-collection.png!
-
-Click on the *Browse...* button and select the files you would like to upload. Selected files will be added to a list of files to be uploaded. After you are done selecting files to upload, click on the * Start* button to start upload. This will start uploading files to Arvados and Workbench will show you the progress bar. When upload is completed, you will see an indication to that effect.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/files-uploaded.png!
-
-*Note:* If you leave the collection page during the upload, the upload process will be aborted and you will need to upload the files again.
-
-*Note:* You can also use the Upload tab to add additional files to an existing collection.
diff --git a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
index 8dcb8e674e..8a08225723 100644
--- a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
@@ -23,13 +23,15 @@ notextile. 

 
 # Start from the *Workbench Dashboard*.  You can access the Dashboard by clicking on * Dashboard* in the upper left corner of any Workbench page.
 # Click on the  Run a process... button.  This will open a dialog box titled *Choose a pipeline or workflow to run*.
-# In the search box, type in *Tutorial bwa mem cwl*.
-# Select * Tutorial bwa mem cwl* and click the Next: choose inputs  button.  This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer.
-# For example, let's see how to change *"reference" parameter* for this workflow. Click the Choose button beneath the *"reference" parameter* header.  This will open a dialog box titled *Choose a dataset for "reference" parameter for cwl-runner in bwa-mem.cwl component*.
-# Open the *Home * menu and select *All Projects*. Search for and select * Tutorial chromosome 19 reference*. You will then see a list of files. Select * 19-fasta.bwt* and click the OK button.
-# Repeat the previous two steps to set the *"read_p1" parameter for cwl-runner script in bwa-mem.cwl component* and *"read_p2" parameter for cwl-runner script in bwa-mem.cwl component* parameters.
-# Click on the Run  button.  The page updates to show you that the process has been submitted to run on the Arvados cluster.
-# After the process starts running, you can track the progress by watching log messages from the component(s).  This page refreshes automatically.  You will see a complete label when the process completes successfully.
+# In the search box, type in *bwa-mem.cwl*.
+# Select * bwa-mem.cwl* and click the Next: choose inputs  button.  This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer.
+# For example, let's see how to set read pair *read_p1* and *read_p2* for this workflow. Click the Choose button beneath the *read_p1* header.  This will open a dialog box titled *Choose a file*.
+# In the file dialog, click on *Home * menu and then select *All Projects*.
+# Enter *HWI-ST1027* into the search box.  You will see one or more collections. Click on *  HWI-ST1027_129_D0THKACXX for CWL tutorial*
+# The right hand panel will list two files.  Click on the first one ending in "_1" and click the OK button.
+# Repeat the steps 5-8 to set the *read_p2* except selecting the second file ending in "_2"
+# Scroll to the bottom of the "Inputs" panel and click on the Run  button.  The page updates to show you that the process has been submitted to run on the Arvados cluster.
+# Once the process starts running, you can track the progress by watching log messages from the component(s).  This page refreshes automatically.  You will see a complete label when the process completes successfully.
 # Click on the *Output* link to see the results of the process.  This will load a new page listing the output files from this process.  You'll see the output SAM file from the alignment tool under the *Files* tab.
 # Click on the  download button to the right of the SAM file to download your results.
 
diff --git a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
index dd537c46ac..0166b8b525 100644
--- a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
+++ b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Writing a CWL workflow"
+title: "Developing workflows with CWL"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -15,7 +15,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2. Developing workflows
 
-For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.1 .
+For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.2 .
 
 See "Writing Portable High-Performance Workflows":{{site.baseurl}}/user/cwl/cwl-style.html and "Arvados CWL Extensions":{{site.baseurl}}/user/cwl/cwl-extensions.html for additional information about using CWL on Arvados.
 
@@ -23,65 +23,6 @@ See "Repositories of CWL Tools and Workflows":https://www.commonwl.org/#Reposito
 
 See "Software for working with CWL":https://www.commonwl.org/#Software_for_working_with_CWL for links to software tools to help create CWL documents.
 
-h2. Using Composer
-
-You can create new workflows in the browser using "Arvados Composer":{{site.baseurl}}/user/composer/composer.html
-
-h2. Registering a workflow to use in Workbench
-
-Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the  Run a process... button on the Workbench Dashboard and on the command line by UUID.
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to qr1hi-4zz18-7e0hedrmkuyoei3
-2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template qr1hi-p5p6p-rjleou1dwr167v5
-qr1hi-p5p6p-rjleou1dwr167v5
-

-
-
-You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to qr1hi-4zz18-0f91qkovk4ml18o
-2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template qr1hi-p5p6p-0deqe6nuuyqns2i
-qr1hi-p5p6p-zuniv58hn8d0qd8
-

-
-
-h3. Running registered workflows at the command line
-
-You can run a registered workflow at the command line by its UUID:
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner qr1hi-p5p6p-zuniv58hn8d0qd8 --help
-/home/peter/work/scripts/venv/bin/arvados-cwl-runner 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59 1.0.20161209192028, arvados-python-client 0.1.20161212125425, cwltool 1.0.20161207161158
-Resolved 'qr1hi-p5p6p-zuniv58hn8d0qd8' to 'keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl'
-usage: qr1hi-p5p6p-zuniv58hn8d0qd8 [-h] [--PL PL] --group_id GROUP_ID
-                                   --read_p1 READ_P1 [--read_p2 READ_P2]
-                                   [--reference REFERENCE] --sample_id
-                                   SAMPLE_ID
-                                   [job_order]
-
-positional arguments:
-  job_order             Job input json file
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --PL PL
-  --group_id GROUP_ID
-  --read_p1 READ_P1     The reads, in fastq format.
-  --read_p2 READ_P2     For mate paired reads, the second file (optional).
-  --reference REFERENCE
-                        The index files produced by `bwa index`
-  --sample_id SAMPLE_ID
-

-
-
 h2. Using cwltool
 
 When developing a workflow, it is often helpful to run it on the local host to avoid the overhead of submitting to the cluster.  To execute a workflow only on the local host (without submitting jobs to an Arvados cluster) you can use the @cwltool@ command.  Note that when using @cwltool@ you must have the input data accessible on the local file system using either @arv-mount@ or @arv-get@ to fetch the data from Keep.
@@ -150,60 +91,3 @@ Final process status is success
 
 
 If you get the error @JavascriptException: Long-running script killed after 20 seconds.@ this may be due to the Dockerized Node.js engine taking too long to start.  You may address this by installing Node.js locally (run @apt-get install nodejs@ on Debian or Ubuntu) or by specifying a longer timeout with the @--eval-timeout@ option.  For example, run the workflow with @cwltool --eval-timeout=40@ for a 40-second timeout.
-
-h2. Making workflows directly executable
-
-You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file:
-
-
-
#!/usr/bin/env cwl-runner
-

-
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-

-
-
-You can even make an input file directly executable the same way with the following two lines at the top:
-
-
-
#!/usr/bin/env cwl-runner
-cwl:tool: bwa-mem.cwl
-

-
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-

-
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 6c47a068e9..80294afaf3 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1314,7 +1314,7 @@ Clusters:
       # a link to the multi-site search page on a "home" Workbench site.
       #
       # Example:
-      #   https://workbench.qr1hi.arvadosapi.com/collections/multisite
+      #   https://workbench.zzzzz.arvadosapi.com/collections/multisite
       MultiSiteSearch: ""
 
       # Should workbench allow management of local git repositories? Set to false if
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index cbb73d97b8..57204cf36a 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -1320,7 +1320,7 @@ Clusters:
       # a link to the multi-site search page on a "home" Workbench site.
       #
       # Example:
-      #   https://workbench.qr1hi.arvadosapi.com/collections/multisite
+      #   https://workbench.zzzzz.arvadosapi.com/collections/multisite
       MultiSiteSearch: ""
 
       # Should workbench allow management of local git repositories? Set to false if
diff --git a/tools/copy-tutorial/copy-tutorial.sh b/tools/copy-tutorial/copy-tutorial.sh
new file mode 100755
index 0000000000..bdc75da2e1
--- /dev/null
+++ b/tools/copy-tutorial/copy-tutorial.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+if test -z "$1"  ; then
+  echo "$0: Copies Arvados tutorial resources from public data cluster (jutro)"
+  echo "Usage: copy-tutorial.sh "
+  echo " is destination cluster configuration that can be found in ~/.config/arvados"
+  exit
+fi
+
+echo "Copying from public data cluster (jutro) to $1"
+
+for a in $(cat $HOME/.config/arvados/$1.conf) ; do export $a ; done
+
+project_uuid=$(arv --format=uuid group create --group '{"name":"User guide resources", "group_class": "project"}')
+
+# Bwa-mem workflow
+arv-copy --src jutro --dst $1 --project-uuid=$project_uuid f141fc27e7cfa7f7b6d208df5e0ee01b+59
+arv-copy --src jutro --dst $1 --project-uuid=$project_uuid jutro-7fd4e-mkmmq53m1ze6apx
+
+echo "Data copied to \"User guide resources\" at $project_uuid"