From 7b6a085a92e78787a15a34ad8e536444ad230552 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 20 Aug 2020 13:46:58 -0400 Subject: [PATCH] Make sure examples work, lots of wordsmithing and reorganizing for clarity. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- doc/_config.yml | 12 +- doc/_includes/_what_is_cwl.liquid | 2 +- doc/sdk/python/events.html.textile.liquid | 2 +- .../cwl/cwl-run-options.html.textile.liquid | 2 +- doc/user/cwl/cwl-runner.html.textile.liquid | 151 ++++++++++++++---- doc/user/cwl/cwl-style.html.textile.liquid | 2 +- doc/user/cwl/cwl-versions.html.textile.liquid | 4 +- .../topics/arv-docker.html.textile.liquid | 28 ++-- .../tutorial-keep-get.html.textile.liquid | 29 ++-- ...l-keep-mount-gnu-linux.html.textile.liquid | 21 ++- ...torial-keep-mount-os-x.html.textile.liquid | 6 +- ...ial-keep-mount-windows.html.textile.liquid | 4 +- ...ial-workflow-workbench.html.textile.liquid | 16 +- .../writing-cwl-workflow.html.textile.liquid | 120 +------------- tools/copy-tutorial/copy-tutorial.sh | 25 +++ 15 files changed, 210 insertions(+), 214 deletions(-) create mode 100755 tools/copy-tutorial/copy-tutorial.sh diff --git a/doc/_config.yml b/doc/_config.yml index 8d31f7e4bb..20b9139986 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -27,7 +27,6 @@ navbar: - Run a workflow using Workbench: - user/getting_started/workbench.html.textile.liquid - user/tutorials/tutorial-workflow-workbench.html.textile.liquid - - user/composer/composer.html.textile.liquid - Working at the Command Line: - user/getting_started/setup-cli.html.textile.liquid - user/reference/api-tokens.html.textile.liquid @@ -47,17 +46,15 @@ navbar: - user/topics/collection-versioning.html.textile.liquid - user/topics/storage-classes.html.textile.liquid - user/topics/keep.html.textile.liquid - - Running workflows at the command line: + - Data Analysis with Workflows: - user/cwl/cwl-runner.html.textile.liquid - user/cwl/cwl-run-options.html.textile.liquid - - Develop an Arvados workflow: - - user/tutorials/intro-crunch.html.textile.liquid - user/tutorials/writing-cwl-workflow.html.textile.liquid + - user/topics/arv-docker.html.textile.liquid - user/cwl/cwl-style.html.textile.liquid - - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-extensions.html.textile.liquid + - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-versions.html.textile.liquid - - user/topics/arv-docker.html.textile.liquid - Working with git repositories: - user/tutorials/add-new-repository.html.textile.liquid - user/tutorials/git-arvados-guide.html.textile.liquid @@ -77,8 +74,9 @@ navbar: - sdk/python/example.html.textile.liquid - sdk/python/python.html.textile.liquid - sdk/python/arvados-fuse.html.textile.liquid - - sdk/python/events.html.textile.liquid + - sdk/python/arvados-cwl-runner.html.textile.liquid - sdk/python/cookbook.html.textile.liquid + - sdk/python/events.html.textile.liquid - CLI: - sdk/cli/install.html.textile.liquid - sdk/cli/index.html.textile.liquid diff --git a/doc/_includes/_what_is_cwl.liquid b/doc/_includes/_what_is_cwl.liquid index fea11b3a32..7fbe7b8156 100644 --- a/doc/_includes/_what_is_cwl.liquid +++ b/doc/_includes/_what_is_cwl.liquid @@ -4,4 +4,4 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -The "Common Workflow Language (CWL)":http://commonwl.org is a multi-vendor open standard for describing analysis tools and workflows that are portable across a variety of platforms. CWL is the primary way to develop and run workflows for Arvados. Arvados supports versions "v1.0":http://commonwl.org/v1.0 and "v1.1":http://commonwl.org/v1.1 of the CWL specification. +The "Common Workflow Language (CWL)":http://commonwl.org is a multi-vendor open standard for describing analysis tools and workflows that are portable across a variety of platforms. CWL is the primary way to develop and run workflows for Arvados. Arvados supports versions "v1.0":http://commonwl.org/v1.0 , "v1.1":http://commonwl.org/v1.1 and "v1.2":http://commonwl.org/v1.2 of the CWL standard. diff --git a/doc/sdk/python/events.html.textile.liquid b/doc/sdk/python/events.html.textile.liquid index afbec20d95..302af20c83 100644 --- a/doc/sdk/python/events.html.textile.liquid +++ b/doc/sdk/python/events.html.textile.liquid @@ -2,7 +2,7 @@ layout: default navsection: sdk navmenu: Python -title: Subscribing to events +title: Subscribing to database events ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid index 0c7de5cb3a..761d198ee4 100644 --- a/doc/user/cwl/cwl-run-options.html.textile.liquid +++ b/doc/user/cwl/cwl-run-options.html.textile.liquid @@ -1,7 +1,7 @@ --- layout: default navsection: userguide -title: "Using arvados-cwl-runner" +title: "arvados-cwl-runner options" ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid index ed481334c1..442a60b04f 100644 --- a/doc/user/cwl/cwl-runner.html.textile.liquid +++ b/doc/user/cwl/cwl-runner.html.textile.liquid @@ -1,7 +1,7 @@ --- layout: default navsection: userguide -title: "Running an Arvados workflow" +title: "Starting a Workflow at the Command Line" ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. @@ -15,9 +15,12 @@ SPDX-License-Identifier: CC-BY-SA-3.0 This tutorial will demonstrate how to submit a workflow at the command line using @arvados-cwl-runner@. -h2. Running arvados-cwl-runner +# "Get the tutorial files":#get-files +# "Submitting a workflow to an Arvados cluster":#submitting +# "Registering a workflow to use in Workbench":#registering +# "Make a workflow file directly executable":#executable -h3. Get the example files +h2(#get-files). Get the tutorial files The tutorial files are located in the documentation section of the Arvados source repository, which can be found on "git.arvados.org":https://git.arvados.org/arvados.git/tree/HEAD:/doc/user/cwl/bwa-mem or "github":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem @@ -41,7 +44,7 @@ If you do not wish to create an account on "https://playground.arvados.org":http "https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/":https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/ -h2. Submitting a workflow to an Arvados cluster +h2(#submitting). Submitting a workflow to an Arvados cluster h3. Submit a workflow and wait for results @@ -79,10 +82,14 @@ If you reference a file in "arv-mount":{{site.baseurl}}/user/tutorials/tutorial- If you reference a local file which is not in @arv-mount@, then @arvados-cwl-runner@ will upload the file to Keep and use the Keep URI reference from the upload. -You can also execute CWL files directly from Keep: +You can also execute CWL files that have been uploaded Keep: -
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl bwa-mem-input.yml
+

+~/arvados/doc/user/cwl/bwa-mem$ arv-put --portable-data-hash --name "bwa-mem.cwl" bwa-mem.cwl
+2020-08-20 13:40:02 arvados.arv_put[12976] INFO: Collection saved as 'bwa-mem.cwl'
+f141fc27e7cfa7f7b6d208df5e0ee01b+59
+~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:f141fc27e7cfa7f7b6d208df5e0ee01b+59/bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
 2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
@@ -100,50 +107,128 @@ arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107,
 

 
 
+Note: uploading a workflow file to Keep is _not_ the same as registering the workflow for use in Workbench.  See "Registering a workflow to use in Workbench":#registering below.
+
 h3. Work reuse
 
 Workflows submitted with @arvados-cwl-runner@ will take advantage of Arvados job reuse.  If you submit a workflow which is identical to one that has run before, it will short cut the execution and return the result of the previous run.  This also applies to individual workflow steps.  For example, a two step workflow where the first step has run before will reuse results for first step and only execute the new second step.  You can disable this behavior with @--disable-reuse@.
 
 h3. Command line options
 
-See "Using arvados-cwl-runner":{{site.baseurl}}/user/cwl/cwl-run-options.html
+See "arvados-cwl-runner options":{{site.baseurl}}/user/cwl/cwl-run-options.html
 
-h2(#setup). Setting up arvados-cwl-runner
+h2(#registering). Registering a workflow to use in Workbench
 
-By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If you want to submit jobs from somewhere else, such as your workstation, you may install @arvados-cwl-runner@ using @pip@:
+Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the  Run a process... button on the Workbench Dashboard and on the command line by UUID.
 
 
-
~$ virtualenv ~/venv
-~$ . ~/venv/bin/activate
-~$ pip install -U setuptools
-~$ pip install arvados-cwl-runner
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
+2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template zzzzz-p5p6p-rjleou1dwr167v5
+zzzzz-p5p6p-rjleou1dwr167v5
 

 
 
-h3. Check Docker access
+You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to zzzzz-4zz18-0f91qkovk4ml18o
+2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template zzzzz-p5p6p-0deqe6nuuyqns2i
+zzzzz-p5p6p-zuniv58hn8d0qd8
+

+
 
-In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Arvados.
+h3. Running registered workflows at the command line
 
-You can determine if you have access to Docker by running @docker version@:
+You can run a registered workflow at the command line by its UUID:
 
 
-
~$ docker version
-Client:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
-
-Server:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner pirca-7fd4e-3nqbw08vtjl8ybz --help
+INFO /home/peter/work/scripts/venv3/bin/arvados-cwl-runner 2.1.0.dev20200814195416, arvados-python-client 2.1.0.dev20200814195416, cwltool 3.0.20200807132242
+INFO Resolved 'pirca-7fd4e-3nqbw08vtjl8ybz' to 'arvwf:pirca-7fd4e-3nqbw08vtjl8ybz#main'
+usage: pirca-7fd4e-3nqbw08vtjl8ybz [-h] [--PL PL] [--group_id GROUP_ID]
+                                   [--read_p1 READ_P1] [--read_p2 READ_P2]
+                                   [--reference REFERENCE]
+                                   [--sample_id SAMPLE_ID]
+                                   [job_order]
+
+positional arguments:
+  job_order             Job input json file
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --PL PL
+  --group_id GROUP_ID
+  --read_p1 READ_P1     The reads, in fastq format.
+  --read_p2 READ_P2     For mate paired reads, the second file (optional).
+  --reference REFERENCE
+                        The index files produced by `bwa index`
+  --sample_id SAMPLE_ID
 

 
 
-If this returns an error, contact the sysadmin of your cluster for assistance.
+h2(#executable). Make a workflow file directly executable
+
+You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file:
+
+
+
#!/usr/bin/env cwl-runner
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+You can even make an input file directly executable the same way with the following two lines at the top:
+
+
+
#!/usr/bin/env cwl-runner
+cwl:tool: bwa-mem.cwl
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+h2(#setup). Setting up arvados-cwl-runner
+
+See "Arvados CWL Runner":{{site.baseurl}}/sdk/python/arvados-cwl-runner.html
diff --git a/doc/user/cwl/cwl-style.html.textile.liquid b/doc/user/cwl/cwl-style.html.textile.liquid
index ee36014cb5..bd07161ce3 100644
--- a/doc/user/cwl/cwl-style.html.textile.liquid
+++ b/doc/user/cwl/cwl-style.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: Writing Portable High-Performance Workflows
+title: Guidelines for Writing High-Performance Portable Workflows
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
diff --git a/doc/user/cwl/cwl-versions.html.textile.liquid b/doc/user/cwl/cwl-versions.html.textile.liquid
index 5fcfcbe3bc..ac679dc154 100644
--- a/doc/user/cwl/cwl-versions.html.textile.liquid
+++ b/doc/user/cwl/cwl-versions.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: CWL version and API support
+title: CWL version support
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,6 +9,8 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
+Arvados supports CWL v1.0, v1.1 and v1.2.
+
 h2(#v12). Upgrading your workflows to CWL v1.2
 
 If you are starting from a CWL v1.0 document, see "Upgrading your workflows to CWL v1.1":#v11 below.
diff --git a/doc/user/topics/arv-docker.html.textile.liquid b/doc/user/topics/arv-docker.html.textile.liquid
index b03ef0bdf1..e021c1819a 100644
--- a/doc/user/topics/arv-docker.html.textile.liquid
+++ b/doc/user/topics/arv-docker.html.textile.liquid
@@ -11,32 +11,30 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 This page describes how to set up the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a workflow step will be run in using "Docker.":https://www.docker.com/  Docker is a tool for building and running containers that isolate applications from other applications running on the same node.  For detailed information about Docker, see the "Docker User Guide.":https://docs.docker.com/userguide/
 
-This page will describe:
+This page describes:
 
-# Create a custom image using a Dockerfile
-# Uploading an image to Arvados
-# Sources of pre-built bioinformatics Docker images
+# "Create a custom image using a Dockerfile":#create
+# "Uploading an image to Arvados":#upload
+# "Sources of pre-built bioinformatics Docker images":#sources
 
 {% include 'tutorial_expectations_workstation' %}
 
 You also need ensure that "Docker is installed,":https://docs.docker.com/installation/ the Docker daemon is running, and you have permission to access Docker.  You can test this by running @docker version@.  If you receive a permission denied error, your user account may need to be added to the @docker@ group.  If you have root access, you can add yourself to the @docker@ group using @$ sudo addgroup $USER docker@ then log out and log back in again; otherwise consult your local sysadmin.
 
-h2. Create a custom image using a Dockerfile
+h2(#create). Create a custom image using a Dockerfile
 
 This example shows how to create a Docker image and add the R package.
 
 First, create new directory called @docker-example@, in that directory create a file called @Dockerfile@.
 
 
-

-$ mkdir docker-example-r-base
-$ cd docker-example-r-base
+
$ mkdir docker-example-r-base
+$ cd docker-example-r-base
 

 
 
 
-

-FROM ubuntu:bionic
+
FROM ubuntu:bionic
 RUN apt-get update && apt-get -yq --no-install-recommends install r-base-core
 

 
@@ -52,8 +50,7 @@ h3. Create a new image
 We're now ready to create a new Docker image.  Use @docker build@ to create a new image from the Dockerfile.
 
 
-

-docker-example-r-base$ docker build -t docker-example-r-base .
+
docker-example-r-base$ docker build -t docker-example-r-base .
 

 
 
@@ -62,17 +59,16 @@ h3. Verify image
 Now we can verify that "R" is installed:
 
 
-
$ docker run -ti docker-example-r-base
+
$ docker run -ti docker-example-r-base
 root@57ec8f8b2663:/# R
 
 R version 3.4.4 (2018-03-15) -- "Someone to Lean On"
 Copyright (C) 2018 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
-
 

 
 
-h2. Upload your image
+h2(#upload). Upload your image
 
 Finally, we are ready to upload the new Docker image to Arvados.  Use @arv-keepdocker@ with the image repository name to upload the image.  Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
 
@@ -107,7 +103,7 @@ Docker images are subject to normal Arvados permissions.  If wish to share your
 

 
 
-h2. Sources of pre-built images
+h2(#sources). Sources of pre-built images
 
 In addition to creating your own contianers, there are a number of resources where you can find bioinformatics tools already wrapped in container images:
 
diff --git a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
index acf4b33c17..05924f8475 100644
--- a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
@@ -52,38 +52,35 @@ You can download Arvados data collections using the command line tools @arv-ls@
 Use @arv-ls@ to view the contents of a collection:
 
 
-
~$ arv-ls c1bad4b39ca5a924e481008009d94e32+210
-var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls ae480c5099b81e17267b7445e35b4bc7+180
+./HWI-ST1027_129_D0THKACXX.1_1.fastq
+./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
-
~$ arv-ls 887cd41e9c613463eab2f0d885c6dd96+83
-alice.txt
-bob.txt
-carol.txt
-

-
-
-Use @-s@ to print file sizes rounded up to the nearest kilobyte:
+Use @-s@ to print file sizes, in kilobytes, rounded up:
 
 
-
~$ arv-ls -s c1bad4b39ca5a924e481008009d94e32+210
-221887 var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls -s ae480c5099b81e17267b7445e35b4bc7+180
+     12258 ./HWI-ST1027_129_D0THKACXX.1_1.fastq
+     12258 ./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 Use @arv-get@ to download the contents of a collection and place it in the directory specified in the second argument (in this example, @.@ for the current directory):
 
 
-
~$ arv-get c1bad4b39ca5a924e481008009d94e32+210/ .
-~$ ls var-GS000016015-ASM.tsv.bz2
-var-GS000016015-ASM.tsv.bz2
+
~$ $ arv-get ae480c5099b81e17267b7445e35b4bc7+180/ .
+23 MiB / 23 MiB 100.0%
+~$ ls
+HWI-ST1027_129_D0THKACXX.1_1.fastq  HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 You can also download individual files:
 
 
-
~$ arv-get 887cd41e9c613463eab2f0d885c6dd96+83/alice.txt .
+
~$ arv-get ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_1.fastq .
+11 MiB / 11 MiB 100.0%
 

 
 
diff --git a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
index eb2a91b757..060ae2acbe 100644
--- a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Keep network mount on GNU/Linux"
+title: "Access Keep as a GNU/Linux filesystem"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,17 +9,16 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-GNU/Linux users can use @arv-mount@ to mount Keep as a file system in order to access Arvados collections using traditional filesystem tools.
+GNU/Linux users can use @arv-mount@ or Gnome to mount Keep as a file system in order to access Arvados collections using traditional filesystem tools.
 
 {% include 'tutorial_expectations' %}
 
-h2. Arv-mount
+# "*Mounting at the command line with arv-mount*":#arv-mount
+# "*Mounting in Gnome File manager*":#gnome
 
-@arv-mount@ provides several features:
+h2(#arv-mount). Arv-mount
 
-* You can browse, open and read Keep entries as if they are regular files.
-* It is easy for existing tools to access files in Keep.
-* Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
+@arv-mount@ provides a file system view of Arvados Keep using File System in Userspace (FUSE).  You can browse, open and read Keep entries as if they are regular files, and existing tools can access files in Keep.  Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
 
 The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory.  To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by UUID or portable data hash. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
 
@@ -59,3 +58,11 @@ Not supported:
 If multiple clients (separate instances of arv-mount or other arvados applications) modify the same file in the same collection within a short time interval, this may result in a conflict.  In this case, the most recent commit wins, and the "loser" will be renamed to a conflict file in the form @name~YYYYMMDD-HHMMSS~conflict~@.
 
 Please note this feature is in beta testing.  In particular, the conflict mechanism is itself currently subject to race conditions with potential for data loss when a collection is being modified simultaneously by multiple clients.  This issue will be resolved in future development.
+
+h2(#gnome). Mounting in Gnome File manager
+
+As an alternative to @arv-mount@ you can also access the WebDAV mount through the Gnome File manager.
+
+# Open "Files"
+# On the left sidebar, click on "Other Locations"
+# At the bottom of the window, enter @davs://collections.ClusterID.example.com/@  When prompted for credentials, enter username "arvados" and a valid Arvados token in the @Password@ field.
diff --git a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
index 736952b5cb..2c8c0b8bb6 100644
--- a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Keep network mount on OS X"
+title: "Access Keep from OS X Finder"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -11,9 +11,9 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 OS X users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in Finder (read-only)
 
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, enter username "arvados" and paste a valid Arvados token for the @Password@ field.
 
 This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
 
diff --git a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
index 0818f632f8..a40a997ba1 100644
--- a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Keep network mount on Windows"
+title: "Access Keep from Windows File Explorer"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -11,7 +11,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Windows users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in File Explorer (read-only)
 
 Use the 'Map network drive' functionality, and enter @https://collections.ClusterID.example.com/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
 
diff --git a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
index 8dcb8e674e..8a08225723 100644
--- a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid
@@ -23,13 +23,15 @@ notextile. 

 
 # Start from the *Workbench Dashboard*.  You can access the Dashboard by clicking on * Dashboard* in the upper left corner of any Workbench page.
 # Click on the  Run a process... button.  This will open a dialog box titled *Choose a pipeline or workflow to run*.
-# In the search box, type in *Tutorial bwa mem cwl*.
-# Select * Tutorial bwa mem cwl* and click the Next: choose inputs  button.  This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer.
-# For example, let's see how to change *"reference" parameter* for this workflow. Click the Choose button beneath the *"reference" parameter* header.  This will open a dialog box titled *Choose a dataset for "reference" parameter for cwl-runner in bwa-mem.cwl component*.
-# Open the *Home * menu and select *All Projects*. Search for and select * Tutorial chromosome 19 reference*. You will then see a list of files. Select * 19-fasta.bwt* and click the OK button.
-# Repeat the previous two steps to set the *"read_p1" parameter for cwl-runner script in bwa-mem.cwl component* and *"read_p2" parameter for cwl-runner script in bwa-mem.cwl component* parameters.
-# Click on the Run  button.  The page updates to show you that the process has been submitted to run on the Arvados cluster.
-# After the process starts running, you can track the progress by watching log messages from the component(s).  This page refreshes automatically.  You will see a complete label when the process completes successfully.
+# In the search box, type in *bwa-mem.cwl*.
+# Select * bwa-mem.cwl* and click the Next: choose inputs  button.  This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer.
+# For example, let's see how to set read pair *read_p1* and *read_p2* for this workflow. Click the Choose button beneath the *read_p1* header.  This will open a dialog box titled *Choose a file*.
+# In the file dialog, click on *Home * menu and then select *All Projects*.
+# Enter *HWI-ST1027* into the search box.  You will see one or more collections. Click on *  HWI-ST1027_129_D0THKACXX for CWL tutorial*
+# The right hand panel will list two files.  Click on the first one ending in "_1" and click the OK button.
+# Repeat the steps 5-8 to set the *read_p2* except selecting the second file ending in "_2"
+# Scroll to the bottom of the "Inputs" panel and click on the Run  button.  The page updates to show you that the process has been submitted to run on the Arvados cluster.
+# Once the process starts running, you can track the progress by watching log messages from the component(s).  This page refreshes automatically.  You will see a complete label when the process completes successfully.
 # Click on the *Output* link to see the results of the process.  This will load a new page listing the output files from this process.  You'll see the output SAM file from the alignment tool under the *Files* tab.
 # Click on the  download button to the right of the SAM file to download your results.
 
diff --git a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
index 4e10eff018..0166b8b525 100644
--- a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
+++ b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Writing a CWL workflow"
+title: "Developing workflows with CWL"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -15,7 +15,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2. Developing workflows
 
-For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.1 .
+For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.2 .
 
 See "Writing Portable High-Performance Workflows":{{site.baseurl}}/user/cwl/cwl-style.html and "Arvados CWL Extensions":{{site.baseurl}}/user/cwl/cwl-extensions.html for additional information about using CWL on Arvados.
 
@@ -23,65 +23,6 @@ See "Repositories of CWL Tools and Workflows":https://www.commonwl.org/#Reposito
 
 See "Software for working with CWL":https://www.commonwl.org/#Software_for_working_with_CWL for links to software tools to help create CWL documents.
 
-h2. Using Composer
-
-You can create new workflows in the browser using "Arvados Composer":{{site.baseurl}}/user/composer/composer.html
-
-h2. Registering a workflow to use in Workbench
-
-Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the  Run a process... button on the Workbench Dashboard and on the command line by UUID.
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
-2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template zzzzz-p5p6p-rjleou1dwr167v5
-zzzzz-p5p6p-rjleou1dwr167v5
-

-
-
-You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to zzzzz-4zz18-0f91qkovk4ml18o
-2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template zzzzz-p5p6p-0deqe6nuuyqns2i
-zzzzz-p5p6p-zuniv58hn8d0qd8
-

-
-
-h3. Running registered workflows at the command line
-
-You can run a registered workflow at the command line by its UUID:
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner zzzzz-p5p6p-zuniv58hn8d0qd8 --help
-/home/peter/work/scripts/venv/bin/arvados-cwl-runner 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59 1.0.20161209192028, arvados-python-client 0.1.20161212125425, cwltool 1.0.20161207161158
-Resolved 'zzzzz-p5p6p-zuniv58hn8d0qd8' to 'keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl'
-usage: zzzzz-p5p6p-zuniv58hn8d0qd8 [-h] [--PL PL] --group_id GROUP_ID
-                                   --read_p1 READ_P1 [--read_p2 READ_P2]
-                                   [--reference REFERENCE] --sample_id
-                                   SAMPLE_ID
-                                   [job_order]
-
-positional arguments:
-  job_order             Job input json file
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --PL PL
-  --group_id GROUP_ID
-  --read_p1 READ_P1     The reads, in fastq format.
-  --read_p2 READ_P2     For mate paired reads, the second file (optional).
-  --reference REFERENCE
-                        The index files produced by `bwa index`
-  --sample_id SAMPLE_ID
-

-
-
 h2. Using cwltool
 
 When developing a workflow, it is often helpful to run it on the local host to avoid the overhead of submitting to the cluster.  To execute a workflow only on the local host (without submitting jobs to an Arvados cluster) you can use the @cwltool@ command.  Note that when using @cwltool@ you must have the input data accessible on the local file system using either @arv-mount@ or @arv-get@ to fetch the data from Keep.
@@ -150,60 +91,3 @@ Final process status is success
 
 
 If you get the error @JavascriptException: Long-running script killed after 20 seconds.@ this may be due to the Dockerized Node.js engine taking too long to start.  You may address this by installing Node.js locally (run @apt-get install nodejs@ on Debian or Ubuntu) or by specifying a longer timeout with the @--eval-timeout@ option.  For example, run the workflow with @cwltool --eval-timeout=40@ for a 40-second timeout.
-
-h2. Making workflows directly executable
-
-You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file:
-
-
-
#!/usr/bin/env cwl-runner
-

-
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-

-
-
-You can even make an input file directly executable the same way with the following two lines at the top:
-
-
-
#!/usr/bin/env cwl-runner
-cwl:tool: bwa-mem.cwl
-

-
-
-
-
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-

-
diff --git a/tools/copy-tutorial/copy-tutorial.sh b/tools/copy-tutorial/copy-tutorial.sh
new file mode 100755
index 0000000000..bdc75da2e1
--- /dev/null
+++ b/tools/copy-tutorial/copy-tutorial.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+if test -z "$1"  ; then
+  echo "$0: Copies Arvados tutorial resources from public data cluster (jutro)"
+  echo "Usage: copy-tutorial.sh "
+  echo " is destination cluster configuration that can be found in ~/.config/arvados"
+  exit
+fi
+
+echo "Copying from public data cluster (jutro) to $1"
+
+for a in $(cat $HOME/.config/arvados/$1.conf) ; do export $a ; done
+
+project_uuid=$(arv --format=uuid group create --group '{"name":"User guide resources", "group_class": "project"}')
+
+# Bwa-mem workflow
+arv-copy --src jutro --dst $1 --project-uuid=$project_uuid f141fc27e7cfa7f7b6d208df5e0ee01b+59
+arv-copy --src jutro --dst $1 --project-uuid=$project_uuid jutro-7fd4e-mkmmq53m1ze6apx
+
+echo "Data copied to \"User guide resources\" at $project_uuid"
-- 
2.30.2