16578: Copy tutorials script WIP

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 5f12b62eebe28bc97a874b907caff735dace3151..dc6cdac885c1b79b5bbd8acb384e36d57087bf0c 100755 (executable)
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -305,7 +305,7 @@ def workflow_collections(obj, locations, docker_images):
             if loc.startswith("keep:"):
                 locations.append(loc[5:])
 
-        docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None)
+        docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None) or obj.get('acrContainerImage', None)
         if docker_image is not None:
             ds = docker_image.split(":", 1)
             tag = ds[1] if len(ds)==2 else 'latest'

diff --git a/tools/copy-tutorial/copy-tutorial.sh b/tools/copy-tutorial/copy-tutorial.sh
index 8d8d2d5e39774c1838e3f6c3be23cff466721f4e..6addeb1cdac62bad70ba6f140d9d013b7001c985 100755 (executable)
--- a/tools/copy-tutorial/copy-tutorial.sh
+++ b/tools/copy-tutorial/copy-tutorial.sh
@@ -5,15 +5,19 @@
 
 set -e -o pipefail
 
-if test -z "$1"  ; then
+if test -z "$1" -o -z "$2"  ; then
   echo "$0: Copies Arvados tutorial resources from public data cluster (jutro)"
-  echo "Usage: copy-tutorial.sh <dest>"
+  echo "Usage: copy-tutorial.sh <dest> <tutorial>"
   echo "<dest> is 5-character cluster id of the destination"
+  echo "<tutorial> is which tutorial to copy, one of:"
+  echo " bwa-mem        Tutorial from https://doc.arvados.org/user/tutorials/tutorial-workflow-workbench.html"
+  echo " whole-genome   Whole genome variant calling tutorial workflow (large)"
   exit
 fi
 
 src=jutro
 dest=$1
+tutorial=$2
 
 if ! test -f $HOME/.config/arvados/${dest}.conf ; then
     echo "Please create $HOME/.config/arvados/${dest}.conf with the following contents:"
@@ -32,27 +36,68 @@ fi
 for a in $(cat $HOME/.config/arvados/${dest}.conf) ; do export $a ; done
 
 echo
-echo "Copying bwa mem example from public data cluster (jutro) to $dest"
+echo "Copying from public data cluster (jutro) to $dest"
 echo
 
-set -x
+make_project() {
+    name="$1"
+    owner="$2"
+    if test -z "$owner" ; then
+       owner=$(arv --format=uuid user current)
+    fi
+    project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "'"$name"'"], ["owner_uuid", "=", "'$owner'"]]')
+    if test -z "$project_uuid" ; then
+       project_uuid=$(arv --format=uuid group create --group '{"name":"'"$name"'", "group_class": "project", "owner_uuid": "'$owner'"}')
+       link=$(arv link create --link '{"link_class": "permission", "name": "can_read", "tail_uuid": "'$dest'-j7d0g-anonymouspublic", "head_uuid": "'$project_uuid'"}')
+    fi
+    echo $project_uuid
+}
 
-project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "User guide resources"]]')
-if test -z "$project_uuid" ; then
-    project_uuid=$(arv --format=uuid group create --group '{"name":"User guide resources", "group_class": "project"}')
-    arv link create --link '{"link_class": "permission", "name": "can_read", "tail_uuid": "'$dest'-j7d0g-anonymouspublic", "head_uuid": "'$project_uuid'"}'
-fi
+copy_jobs_image() {
+    if ! arv-keepdocker | grep "arvados/jobs *latest" ; then
+       arv-copy --src jutro --dst $dest --project-uuid=$project_uuid jutro-4zz18-sxmit0qs6i9n2s4
+    fi
+}
+
+parent_project=$(make_project "Tutorial projects")
+copy_jobs_image
+
+if test "$tutorial" = "bwa-mem" ; then
+    echo
+    echo "Copying bwa mem tutorial"
+    echo
+
+    set -x
+
+    project_uuid=$(make_project 'User guide resources' $parent_project)
+
+    # Bwa-mem workflow
+    arv-copy --src jutro --dst $dest --project-uuid=$project_uuid jutro-7fd4e-mkmmq53m1ze6apx
+
+    set +x
 
-if ! arv-keepdocker | grep "arvados/jobs *latest" ; then
-    arv-copy --src jutro --dst $dest --project-uuid=$project_uuid jutro-4zz18-sxmit0qs6i9n2s4
+    echo
+    echo "Finished, data copied to \"User guide resources\" at $project_uuid"
+    echo "You can now go to Workbench and choose 'Run a process' and then select 'bwa-mem.cwl'"
+    echo
 fi
 
-# Bwa-mem workflow
-arv-copy --src jutro --dst $dest --project-uuid=$project_uuid f141fc27e7cfa7f7b6d208df5e0ee01b+59
-arv-copy --src jutro --dst $dest --project-uuid=$project_uuid jutro-7fd4e-mkmmq53m1ze6apx
+if test "$tutorial" = "whole-genome" ; then
+    echo
+    echo "Copying whole genome variant calling tutorial"
+    echo
 
-set +x
+    set -x
 
-echo
-echo "Finished, data copied to \"User guide resources\" at $project_uuid"
-echo
+    project_uuid=$(make_project 'WGS Processing Tutorial' $parent_project)
+
+    # WGS workflow
+    arv-copy --src jutro --dst $dest --project-uuid=$project_uuid jutro-7fd4e-tnxg9ytblbxm26i
+
+    set +x
+
+    echo
+    echo "Finished, data copied to \"WGS Processing Tutorial\" at $project_uuid"
+    echo "You can now go to Workbench and choose 'Run a process' and then select 'bwa-mem.cwl'"
+    echo
+fi