From: Tom Clegg Date: Tue, 15 Sep 2020 14:49:44 +0000 (-0400) Subject: Merge branch '16834-stale-run-lock' X-Git-Tag: 2.1.0~63 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/d6598fd6339e6219a7103781433356dfde546527?hp=6e56f24fcafcefbe6209a9ef53d784aef16c9711 Merge branch '16834-stale-run-lock' fixes #16834 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py index 5f12b62eeb..93fd6b598a 100755 --- a/sdk/python/arvados/commands/arv_copy.py +++ b/sdk/python/arvados/commands/arv_copy.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# arv-copy [--recursive] [--no-recursive] object-uuid src dst +# arv-copy [--recursive] [--no-recursive] object-uuid # # Copies an object from Arvados instance src to instance dst. # @@ -34,6 +34,7 @@ import sys import logging import tempfile import urllib.parse +import io import arvados import arvados.config @@ -87,17 +88,17 @@ def main(): '-f', '--force', dest='force', action='store_true', help='Perform copy even if the object appears to exist at the remote destination.') copy_opts.add_argument( - '--src', dest='source_arvados', required=True, + '--src', dest='source_arvados', help='The name of the source Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.') copy_opts.add_argument( - '--dst', dest='destination_arvados', required=True, + '--dst', dest='destination_arvados', help='The name of the destination Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.') copy_opts.add_argument( '--recursive', dest='recursive', action='store_true', - help='Recursively copy any dependencies for this object. (default)') + help='Recursively copy any dependencies for this object, and subprojects. (default)') copy_opts.add_argument( '--no-recursive', dest='recursive', action='store_false', - help='Do not copy any dependencies. NOTE: if this option is given, the copied object will need to be updated manually in order to be functional.') + help='Do not copy any dependencies or subprojects.') copy_opts.add_argument( '--project-uuid', dest='project_uuid', help='The UUID of the project at the destination to which the collection or workflow should be copied.') @@ -118,6 +119,9 @@ def main(): else: logger.setLevel(logging.INFO) + if not args.source_arvados: + args.source_arvados = args.object_uuid[:5] + # Create API clients for the source and destination instances src_arv = api_for_instance(args.source_arvados) dst_arv = api_for_instance(args.destination_arvados) @@ -135,6 +139,9 @@ def main(): elif t == 'Workflow': set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args) result = copy_workflow(args.object_uuid, src_arv, dst_arv, args) + elif t == 'Group': + set_src_owner_uuid(src_arv.groups(), args.object_uuid, args) + result = copy_project(args.object_uuid, src_arv, dst_arv, args.project_uuid, args) else: abort("cannot copy object {} of type {}".format(args.object_uuid, t)) @@ -170,6 +177,10 @@ def set_src_owner_uuid(resource, uuid, args): # $HOME/.config/arvados/instance_name.conf # def api_for_instance(instance_name): + if not instance_name: + # Use environment + return arvados.api('v1', model=OrderedJsonModel()) + if '/' in instance_name: config_file = instance_name else: @@ -296,7 +307,14 @@ def copy_workflow(wf_uuid, src, dst, args): # copy the workflow itself del wf['uuid'] wf['owner_uuid'] = args.project_uuid - return dst.workflows().create(body=wf).execute(num_retries=args.retries) + + existing = dst.workflows().list(filters=[["owner_uuid", "=", args.project_uuid], + ["name", "=", wf["name"]]]).execute() + if len(existing["items"]) == 0: + return dst.workflows().create(body=wf).execute(num_retries=args.retries) + else: + return dst.workflows().update(uuid=existing["items"][0]["uuid"], body=wf).execute(num_retries=args.retries) + def workflow_collections(obj, locations, docker_images): if isinstance(obj, dict): @@ -305,7 +323,7 @@ def workflow_collections(obj, locations, docker_images): if loc.startswith("keep:"): locations.append(loc[5:]) - docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None) + docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None) or obj.get('acrContainerImage', None) if docker_image is not None: ds = docker_image.split(":", 1) tag = ds[1] if len(ds)==2 else 'latest' @@ -516,7 +534,7 @@ def copy_collection(obj_uuid, src, dst, args): # a new manifest as we go. src_keep = arvados.keep.KeepClient(api_client=src, num_retries=args.retries) dst_keep = arvados.keep.KeepClient(api_client=dst, num_retries=args.retries) - dst_manifest = "" + dst_manifest = io.StringIO() dst_locators = {} bytes_written = 0 bytes_expected = total_collection_size(manifest) @@ -527,14 +545,15 @@ def copy_collection(obj_uuid, src, dst, args): for line in manifest.splitlines(): words = line.split() - dst_manifest += words[0] + dst_manifest.write(words[0]) for word in words[1:]: try: loc = arvados.KeepLocator(word) except ValueError: # If 'word' can't be parsed as a locator, # presume it's a filename. - dst_manifest += ' ' + word + dst_manifest.write(' ') + dst_manifest.write(word) continue blockhash = loc.md5sum # copy this block if we haven't seen it before @@ -547,17 +566,18 @@ def copy_collection(obj_uuid, src, dst, args): dst_locator = dst_keep.put(data) dst_locators[blockhash] = dst_locator bytes_written += loc.size - dst_manifest += ' ' + dst_locators[blockhash] - dst_manifest += "\n" + dst_manifest.write(' ') + dst_manifest.write(dst_locators[blockhash]) + dst_manifest.write("\n") if progress_writer: progress_writer.report(obj_uuid, bytes_written, bytes_expected) progress_writer.finish() # Copy the manifest and save the collection. - logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest) + logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest.getvalue()) - c['manifest_text'] = dst_manifest + c['manifest_text'] = dst_manifest.getvalue() return create_collection_from(c, src, dst, args) def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_http_opt): @@ -632,6 +652,42 @@ def copy_docker_image(docker_image, docker_image_tag, src, dst, args): else: logger.warning('Could not find docker image {}:{}'.format(docker_image, docker_image_tag)) +def copy_project(obj_uuid, src, dst, owner_uuid, args): + + src_project_record = src.groups().get(uuid=obj_uuid).execute(num_retries=args.retries) + + # Create/update the destination project + existing = dst.groups().list(filters=[["owner_uuid", "=", owner_uuid], + ["name", "=", src_project_record["name"]]]).execute(num_retries=args.retries) + if len(existing["items"]) == 0: + project_record = dst.groups().create(body={"group": {"group_class": "project", + "owner_uuid": owner_uuid, + "name": src_project_record["name"]}}).execute(num_retries=args.retries) + else: + project_record = existing["items"][0] + + dst.groups().update(uuid=project_record["uuid"], + body={"group": { + "description": src_project_record["description"]}}).execute(num_retries=args.retries) + + args.project_uuid = project_record["uuid"] + + logger.debug('Copying %s to %s', obj_uuid, project_record["uuid"]) + + # Copy collections + copy_collections([col["uuid"] for col in arvados.util.list_all(src.collections().list, filters=[["owner_uuid", "=", obj_uuid]])], + src, dst, args) + + # Copy workflows + for w in arvados.util.list_all(src.workflows().list, filters=[["owner_uuid", "=", obj_uuid]]): + copy_workflow(w["uuid"], src, dst, args) + + if args.recursive: + for g in arvados.util.list_all(src.groups().list, filters=[["owner_uuid", "=", obj_uuid]]): + copy_project(g["uuid"], src, dst, project_record["uuid"], args) + + return project_record + # git_rev_parse(rev, repo) # # Returns the 40-character commit hash corresponding to 'rev' in @@ -654,7 +710,7 @@ def git_rev_parse(rev, repo): # Special case: if handed a Keep locator hash, return 'Collection'. # def uuid_type(api, object_uuid): - if re.match(r'^[a-f0-9]{32}\+[0-9]+(\+[A-Za-z0-9+-]+)?$', object_uuid): + if re.match(arvados.util.keep_locator_pattern, object_uuid): return 'Collection' p = object_uuid.split('-') if len(p) == 3: diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index f4c1230cc8..0cb4151ac3 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -43,6 +43,14 @@ import arvados.config ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..')) SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services') + +# Work around https://bugs.python.org/issue27805, should be no longer +# necessary from sometime in Python 3.8.x +if not os.environ.get('ARVADOS_DEBUG', ''): + WRITE_MODE = 'a' +else: + WRITE_MODE = 'w' + if 'GOPATH' in os.environ: # Add all GOPATH bin dirs to PATH -- but insert them after the # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler @@ -327,7 +335,7 @@ def run(leave_running_atexit=False): env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) - logf = open(_logfilename('railsapi'), 'a') + logf = open(_logfilename('railsapi'), WRITE_MODE) railsapi = subprocess.Popen( ['bundle', 'exec', 'passenger', 'start', '-p{}'.format(port), @@ -409,7 +417,7 @@ def run_controller(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: return stop_controller() - logf = open(_logfilename('controller'), 'a') + logf = open(_logfilename('controller'), WRITE_MODE) port = internal_port_from_config("Controller") controller = subprocess.Popen( ["arvados-server", "controller"], @@ -429,7 +437,7 @@ def run_ws(): return stop_ws() port = internal_port_from_config("Websocket") - logf = open(_logfilename('ws'), 'a') + logf = open(_logfilename('ws'), WRITE_MODE) ws = subprocess.Popen( ["arvados-server", "ws"], stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -462,7 +470,7 @@ def _start_keep(n, blob_signing=False): yaml.safe_dump(confdata, f) keep_cmd = ["keepstore", "-config", conf] - with open(_logfilename('keep{}'.format(n)), 'a') as logf: + with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf: with open('/dev/null') as _stdin: child = subprocess.Popen( keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True) @@ -529,7 +537,7 @@ def run_keep_proxy(): port = internal_port_from_config("Keepproxy") env = os.environ.copy() env['ARVADOS_API_TOKEN'] = auth_token('anonymous') - logf = open(_logfilename('keepproxy'), 'a') + logf = open(_logfilename('keepproxy'), WRITE_MODE) kp = subprocess.Popen( ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -568,7 +576,7 @@ def run_arv_git_httpd(): gitport = internal_port_from_config("GitHTTP") env = os.environ.copy() env.pop('ARVADOS_API_TOKEN', None) - logf = open(_logfilename('arv-git-httpd'), 'a') + logf = open(_logfilename('arv-git-httpd'), WRITE_MODE) agh = subprocess.Popen(['arv-git-httpd'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('arv-git-httpd'), 'w') as f: @@ -587,7 +595,7 @@ def run_keep_web(): keepwebport = internal_port_from_config("WebDAV") env = os.environ.copy() - logf = open(_logfilename('keep-web'), 'a') + logf = open(_logfilename('keep-web'), WRITE_MODE) keepweb = subprocess.Popen( ['keep-web'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py index 324d6e05d7..452c2beba2 100644 --- a/sdk/python/tests/test_arv_copy.py +++ b/sdk/python/tests/test_arv_copy.py @@ -7,11 +7,18 @@ import os import sys import tempfile import unittest +import shutil +import arvados.api +from arvados.collection import Collection, CollectionReader import arvados.commands.arv_copy as arv_copy from . import arvados_testutil as tutil +from . import run_test_server + +class ArvCopyVersionTestCase(run_test_server.TestCaseWithServers, tutil.VersionChecker): + MAIN_SERVER = {} + KEEP_SERVER = {} -class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker): def run_copy(self, args): sys.argv = ['arv-copy'] + args return arv_copy.main() @@ -26,3 +33,50 @@ class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker): with self.assertRaises(SystemExit): self.run_copy(['--version']) self.assertVersionOutput(out, err) + + def test_copy_project(self): + api = arvados.api() + src_proj = api.groups().create(body={"group": {"name": "arv-copy project", "group_class": "project"}}).execute()["uuid"] + + c = Collection() + with c.open('foo', 'wt') as f: + f.write('foo') + c.save_new("arv-copy foo collection", owner_uuid=src_proj) + + dest_proj = api.groups().create(body={"group": {"name": "arv-copy dest project", "group_class": "project"}}).execute()["uuid"] + + tmphome = tempfile.mkdtemp() + home_was = os.environ['HOME'] + os.environ['HOME'] = tmphome + try: + cfgdir = os.path.join(tmphome, ".config", "arvados") + os.makedirs(cfgdir) + with open(os.path.join(cfgdir, "zzzzz.conf"), "wt") as f: + f.write("ARVADOS_API_HOST=%s\n" % os.environ["ARVADOS_API_HOST"]) + f.write("ARVADOS_API_TOKEN=%s\n" % os.environ["ARVADOS_API_TOKEN"]) + f.write("ARVADOS_API_HOST_INSECURE=1\n") + + contents = api.groups().list(filters=[["owner_uuid", "=", dest_proj]]).execute() + assert len(contents["items"]) == 0 + + try: + self.run_copy(["--project-uuid", dest_proj, src_proj]) + except SystemExit as e: + assert e.code == 0 + + contents = api.groups().list(filters=[["owner_uuid", "=", dest_proj]]).execute() + assert len(contents["items"]) == 1 + + assert contents["items"][0]["name"] == "arv-copy project" + copied_project = contents["items"][0]["uuid"] + + contents = api.collections().list(filters=[["owner_uuid", "=", copied_project]]).execute() + assert len(contents["items"]) == 1 + + assert contents["items"][0]["uuid"] != c.manifest_locator() + assert contents["items"][0]["name"] == "arv-copy foo collection" + assert contents["items"][0]["portable_data_hash"] == c.portable_data_hash() + + finally: + os.environ['HOME'] = home_was + shutil.rmtree(tmphome) diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb index 867b9a6e6a..cd23706d08 100644 --- a/services/api/app/controllers/arvados/v1/users_controller.rb +++ b/services/api/app/controllers/arvados/v1/users_controller.rb @@ -22,7 +22,15 @@ class Arvados::V1::UsersController < ApplicationController rescue ActiveRecord::RecordNotUnique retry end - u.update_attributes!(nullify_attrs(attrs)) + needupdate = {} + nullify_attrs(attrs).each do |k,v| + if !v.nil? && u.send(k) != v + needupdate[k] = v + end + end + if needupdate.length > 0 + u.update_attributes!(needupdate) + end @objects << u end @offset = 0 diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb index 0ce9f1137f..ea5d5b1436 100644 --- a/services/api/test/functional/arvados/v1/users_controller_test.rb +++ b/services/api/test/functional/arvados/v1/users_controller_test.rb @@ -1039,9 +1039,12 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase test "batch update" do existinguuid = 'remot-tpzed-foobarbazwazqux' newuuid = 'remot-tpzed-newnarnazwazqux' + unchanginguuid = 'remot-tpzed-nochangingattrs' act_as_system_user do User.create!(uuid: existinguuid, email: 'root@existing.example.com') + User.create!(uuid: unchanginguuid, email: 'root@unchanging.example.com', prefs: {'foo' => {'bar' => 'baz'}}) end + assert_equal(1, Log.where(object_uuid: unchanginguuid).count) authorize_with(:admin) patch(:batch_update, @@ -1059,6 +1062,10 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase 'email' => 'root@remot.example.com', 'username' => '', }, + unchanginguuid => { + 'email' => 'root@unchanging.example.com', + 'prefs' => {'foo' => {'bar' => 'baz'}}, + }, }}) assert_response(:success) @@ -1070,6 +1077,8 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase assert_equal('noot', User.find_by_uuid(newuuid).first_name) assert_equal('root@remot.example.com', User.find_by_uuid(newuuid).email) + + assert_equal(1, Log.where(object_uuid: unchanginguuid).count) end NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name", diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base index bde5ffe898..815db22b4c 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base @@ -10,18 +10,20 @@ RUN apt-get update && \ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ postgresql-9.6 postgresql-contrib-9.6 git build-essential runit curl libpq-dev \ libcurl4-openssl-dev libssl1.0-dev zlib1g-dev libpcre3-dev libpam-dev \ - openssh-server python-setuptools netcat-traditional \ - python-epydoc graphviz bzip2 less sudo virtualenv \ - libpython-dev fuse libfuse-dev python-pip python-yaml \ - pkg-config libattr1-dev python-pycurl \ + openssh-server netcat-traditional \ + graphviz bzip2 less sudo virtualenv \ + libpython-dev fuse libfuse-dev \ + pkg-config libattr1-dev \ libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \ libjson-perl nginx gitolite3 lsof libreadline-dev \ - apt-transport-https ca-certificates \ - linkchecker python3-virtualenv python-virtualenv xvfb iceweasel \ + apt-transport-https ca-certificates python3-yaml \ + linkchecker python3-virtualenv python3-venv xvfb iceweasel \ libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \ libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \ python3-setuptools python3-pip openjdk-8-jdk bsdmainutils net-tools \ - ruby2.3 ruby-dev bundler shellinabox && \ + ruby2.3 ruby-dev bundler shellinabox && \ + apt-get remove -yq libpython-dev libpython-stdlib libpython2.7 libpython2.7-dev \ + libpython2.7-minimal libpython2.7-stdlib python2.7-minimal python2.7 && \ apt-get clean ENV RUBYVERSION_MINOR 2.3 @@ -78,8 +80,6 @@ ENV GDVERSION=v0.23.0 ENV GDURL=https://github.com/mozilla/geckodriver/releases/download/$GDVERSION/geckodriver-$GDVERSION-linux64.tar.gz RUN set -e && curl -L -f ${GDURL} | tar -C /usr/local/bin -xzf - geckodriver -RUN pip install -U setuptools - ENV NODEVERSION v8.15.1 # Install nodejs binary diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service index 8a36140bcf..1ec225ca12 100755 --- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service @@ -37,13 +37,13 @@ fi pip_install wheel cd /usr/src/arvados/sdk/python -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1) cd /usr/src/arvados/services/fuse -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1) cd /usr/src/arvados/sdk/cwl -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1) diff --git a/tools/arvbox/lib/arvbox/docker/yml_override.py b/tools/arvbox/lib/arvbox/docker/yml_override.py index 446448f5eb..7f35ac1d68 100755 --- a/tools/arvbox/lib/arvbox/docker/yml_override.py +++ b/tools/arvbox/lib/arvbox/docker/yml_override.py @@ -20,7 +20,7 @@ with open(fn) as f: def recursiveMerge(a, b): if isinstance(a, dict) and isinstance(b, dict): for k in b: - print k + print(k) a[k] = recursiveMerge(a.get(k), b[k]) return a else: diff --git a/tools/copy-tutorial/copy-tutorial.sh b/tools/copy-tutorial/copy-tutorial.sh index bdc75da2e1..e7fac7af48 100755 --- a/tools/copy-tutorial/copy-tutorial.sh +++ b/tools/copy-tutorial/copy-tutorial.sh @@ -1,25 +1,83 @@ -#!/bin/sh +#!/bin/bash # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 -set -e +set -e -o pipefail -if test -z "$1" ; then +if test -z "$1" ; then echo "$0: Copies Arvados tutorial resources from public data cluster (jutro)" - echo "Usage: copy-tutorial.sh " - echo " is destination cluster configuration that can be found in ~/.config/arvados" + echo "Usage: copy-tutorial.sh " + echo " is which tutorial to copy, one of:" + echo " bwa-mem Tutorial from https://doc.arvados.org/user/tutorials/tutorial-workflow-workbench.html" + echo " whole-genome Whole genome variant calling tutorial workflow (large)" exit fi -echo "Copying from public data cluster (jutro) to $1" +if test -z "ARVADOS_API_HOST" ; then + echo "Please set ARVADOS_API_HOST to the destination cluster" + exit +fi + +src=jutro +tutorial=$1 + +if ! test -f $HOME/.config/arvados/jutro.conf ; then + # Set it up with the anonymous user token. + echo "ARVADOS_API_HOST=jutro.arvadosapi.com" > $HOME/.config/arvados/jutro.conf + echo "ARVADOS_API_TOKEN=v2/jutro-gj3su-e2o9x84aeg7q005/22idg1m3zna4qe4id3n0b9aw86t72jdw8qu1zj45aboh1mm4ej" >> $HOME/.config/arvados/jutro.conf + exit 1 +fi + +echo +echo "Copying from public data cluster (jutro) to $ARVADOS_API_HOST" +echo + +make_project() { + name="$1" + owner="$2" + if test -z "$owner" ; then + owner=$(arv --format=uuid user current) + fi + project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "'"$name"'"], ["owner_uuid", "=", "'$owner'"]]') + if test -z "$project_uuid" ; then + project_uuid=$(arv --format=uuid group create --group '{"name":"'"$name"'", "group_class": "project", "owner_uuid": "'$owner'"}') + + fi + echo $project_uuid +} -for a in $(cat $HOME/.config/arvados/$1.conf) ; do export $a ; done +copy_jobs_image() { + if ! arv-keepdocker | grep "arvados/jobs *latest" ; then + arv-copy --project-uuid=$parent_project jutro-4zz18-sxmit0qs6i9n2s4 + fi +} -project_uuid=$(arv --format=uuid group create --group '{"name":"User guide resources", "group_class": "project"}') +parent_project=$(make_project "Tutorial projects") +copy_jobs_image -# Bwa-mem workflow -arv-copy --src jutro --dst $1 --project-uuid=$project_uuid f141fc27e7cfa7f7b6d208df5e0ee01b+59 -arv-copy --src jutro --dst $1 --project-uuid=$project_uuid jutro-7fd4e-mkmmq53m1ze6apx +if test "$tutorial" = "bwa-mem" ; then + echo + echo "Copying bwa mem tutorial" + echo -echo "Data copied to \"User guide resources\" at $project_uuid" + arv-copy --project-uuid=$parent_project jutro-j7d0g-rehmt1w5v2p2drp + + echo + echo "Finished, data copied to \"User guide resources\" at $parent_project" + echo "You can now go to Workbench and choose 'Run a process' and then select 'bwa-mem.cwl'" + echo +fi + +if test "$tutorial" = "whole-genome" ; then + echo + echo "Copying whole genome variant calling tutorial" + echo + + arv-copy --project-uuid=$parent_project jutro-j7d0g-n2g87m02rsl4cx2 + + echo + echo "Finished, data copied to \"WGS Processing Tutorial\" at $parent_project" + echo "You can now go to Workbench and choose 'Run a process' and then select 'WGS Processing Tutorial'" + echo +fi