X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9ba66dcc4f4c166fa08512ce0c75e324f26c7b0d..31c82149c41a5c2c1377b617bfedc0dfbb08faf4:/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
diff --git a/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid b/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
index d128b4b195..fa63588730 100644
--- a/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
@@ -1,29 +1,28 @@
---
layout: default
navsection: userguide
-navmenu: Tutorials
-title: "Constructing a Crunch pipeline"
-
+title: "Writing a multi-step pipeline"
...
-h1. Constructing a Crunch pipeline
-
A pipeline in Arvados is a collection of crunch scripts, in which the output from one script may be used as the input to another script.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
+This tutorial uses *@you@* to denote your username. Replace *@you@* with your user name in all the following examples.
+
h2. Create a new script
-Our second script will filter the output of @parallel_hash.py@ and only include hashes that start with 0. Create a new script in @crunch_scripts/@ called @0-filter.py@:
+Our second script will filter the output of @hash.py@ and only include hashes that start with 0. Create a new script in ~/you/crunch_scripts/
{% include '0_filter_py' %}
+$ git add 0-filter.py
-$ git commit -m"zero filter"
-$ git push origin master
+~/you/crunch_scripts$ chmod +x 0-filter.py
+~/you/crunch_scripts$ git add 0-filter.py
+~/you/crunch_scripts$ git commit -m"zero filter"
+~/you/crunch_scripts$ git push origin master
$ cat >the_pipeline <<EOF
+~/you/crunch_scripts$ cat >~/the_pipeline <<EOF
{
- "name":"my_first_pipeline",
+ "name":"Filter md5 hash values",
"components":{
"do_hash":{
- "script":"parallel-hash.py",
+ "script":"hash.py",
"script_parameters":{
- "input": "887cd41e9c613463eab2f0d885c6dd96+83"
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
},
- "script_version":"you:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":false
},
- "filter":{
+ "do_filter":{
"script":"0-filter.py",
"script_parameters":{
"input":{
"output_of":"do_hash"
}
},
- "script_version":"you:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true
}
}
}
EOF
-
+
$ arv pipeline_template create --pipeline-template "$(cat the_pipeline)"
-qr1hi-p5p6p-xxxxxxxxxxxxxxx
-
-$ arv pipeline run --template qr1hi-p5p6p-xxxxxxxxxxxxxxx
-2013-12-16 14:08:40 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 queued 2013-12-16T14:08:40Z
-filter - -
-2013-12-16 14:08:51 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 e2ccd204bca37c77c0ba59fc470cd0f7+162
-filter qr1hi-8i9sb-w5k40fztqgg9i2x queued 2013-12-16T14:08:50Z
-2013-12-16 14:09:01 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 e2ccd204bca37c77c0ba59fc470cd0f7+162
-filter qr1hi-8i9sb-w5k40fztqgg9i2x 735ac35adf430126cf836547731f3af6+56
-
-$ arv keep get e2ccd204bca37c77c0ba59fc470cd0f7+162/md5sum.txt
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
-504938460ef369cd275e4ef58994cffe bob.txt
-8f3b36aff310e06f3c5b9e95678ff77a carol.txt
-$ arv keep get 735ac35adf430126cf836547731f3af6+56
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
-
-...
- "do_hash":{
- "script_parameters":{
- "input": "887cd41e9c613463eab2f0d885c6dd96+83"
- },
- }
-...
-
-$ arv pipeline run --template qr1hi-d1hrv-vxzkp38nlde9yyr do_hash::input=33a9f3842b01ea3fdf27cc582f5ea2af+242
-2013-12-17 20:31:24 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 queued 2013-12-17T20:31:24Z
-filter - -
-2013-12-17 20:31:34 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 {:done=>1, :running=>1, :failed=>0, :todo=>0}
-filter - -
-2013-12-17 20:31:44 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 {:done=>1, :running=>1, :failed=>0, :todo=>0}
-filter - -
-2013-12-17 20:31:55 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
-filter qr1hi-8i9sb-j347g1sqovdh0op queued 2013-12-17T20:31:55Z
-2013-12-17 20:32:05 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
-filter qr1hi-8i9sb-j347g1sqovdh0op fb728f0ffe152058fa64b9aeed344cb5+54
-
-$ arv keep ls -s fb728f0ffe152058fa64b9aeed344cb5+54
-0 0-filter.txt
+~/you/crunch_scripts$ arv pipeline_template create --pipeline-template "$(cat ~/the_pipeline)"