projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch '8624-fuse-tags' closes #8624
[arvados.git]
/
sdk
/
python
/
arvados
/
commands
/
arv_copy.py
diff --git
a/sdk/python/arvados/commands/arv_copy.py
b/sdk/python/arvados/commands/arv_copy.py
index 71820b3d2eae8b64377e0b185ea73c6cb2720f84..8850d0bfd5a82c10633a3b39d0d5958961ead940 100755
(executable)
--- a/
sdk/python/arvados/commands/arv_copy.py
+++ b/
sdk/python/arvados/commands/arv_copy.py
@@
-1,4
+1,6
@@
-#! /usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
# arv-copy [--recursive] [--no-recursive] object-uuid src dst
#
# arv-copy [--recursive] [--no-recursive] object-uuid src dst
#
@@
-16,6
+18,12
@@
# instances src and dst. If either of these files is not found,
# arv-copy will issue an error.
# instances src and dst. If either of these files is not found,
# arv-copy will issue an error.
+from __future__ import division
+from future import standard_library
+from future.utils import listvalues
+standard_library.install_aliases()
+from past.builtins import basestring
+from builtins import object
import argparse
import contextlib
import getpass
import argparse
import contextlib
import getpass
@@
-25,7
+33,7
@@
import shutil
import sys
import logging
import tempfile
import sys
import logging
import tempfile
-import urlparse
+import url
lib.
parse
import arvados
import arvados.config
import arvados
import arvados.config
@@
-113,7
+121,7
@@
def main():
copy_opts.set_defaults(recursive=True)
parser = argparse.ArgumentParser(
copy_opts.set_defaults(recursive=True)
parser = argparse.ArgumentParser(
- description='Copy a pipeline instance, template or collection from one Arvados instance to another.',
+ description='Copy a pipeline instance, template
, workflow,
or collection from one Arvados instance to another.',
parents=[copy_opts, arv_cmd.retry_opt])
args = parser.parse_args()
parents=[copy_opts, arv_cmd.retry_opt])
args = parser.parse_args()
@@
-152,7
+160,7
@@
def main():
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
# Clean up any outstanding temp git repositories.
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
# Clean up any outstanding temp git repositories.
- for d in l
ocal_repo_dir.values(
):
+ for d in l
istvalues(local_repo_dir
):
shutil.rmtree(d, ignore_errors=True)
# If no exception was thrown and the response does not have an
shutil.rmtree(d, ignore_errors=True)
# If no exception was thrown and the response does not have an
@@
-344,7
+352,7
@@
def migrate_components_filters(template_components, dst_git_repo):
be None if that is not known.
"""
errors = []
be None if that is not known.
"""
errors = []
- for cname, cspec in template_components.ite
rite
ms():
+ for cname, cspec in template_components.items():
def add_error(errmsg):
errors.append("{}: {}".format(cname, errmsg))
if not isinstance(cspec, dict):
def add_error(errmsg):
errors.append("{}: {}".format(cname, errmsg))
if not isinstance(cspec, dict):
@@
-425,33
+433,47
@@
def copy_workflow(wf_uuid, src, dst, args):
# fetch the workflow from the source instance
wf = src.workflows().get(uuid=wf_uuid).execute(num_retries=args.retries)
# fetch the workflow from the source instance
wf = src.workflows().get(uuid=wf_uuid).execute(num_retries=args.retries)
+ # copy collections and docker images
if args.recursive:
wf_def = yaml.safe_load(wf["definition"])
if wf_def is not None:
if args.recursive:
wf_def = yaml.safe_load(wf["definition"])
if wf_def is not None:
- colls = []
+ locations = []
+ docker_images = {}
graph = wf_def.get('$graph', None)
if graph is not None:
graph = wf_def.get('$graph', None)
if graph is not None:
- workflow_collections(graph,
coll
s)
+ workflow_collections(graph,
locations, docker_image
s)
else:
else:
- workflow_collections(wf_def, colls)
- copy_collections(colls, src, dst, args)
+ workflow_collections(wf_def, locations, docker_images)
+
+ if locations:
+ copy_collections(locations, src, dst, args)
+ for image in docker_images:
+ copy_docker_image(image, docker_images[image], src, dst, args)
+
+ # copy the workflow itself
del wf['uuid']
wf['owner_uuid'] = args.project_uuid
del wf['uuid']
wf['owner_uuid'] = args.project_uuid
-
return dst.workflows().create(body=wf).execute(num_retries=args.retries)
return dst.workflows().create(body=wf).execute(num_retries=args.retries)
-def workflow_collections(obj,
coll
s):
+def workflow_collections(obj,
locations, docker_image
s):
if isinstance(obj, dict):
loc = obj.get('location', None)
if loc is not None:
if loc.startswith("keep:"):
if isinstance(obj, dict):
loc = obj.get('location', None)
if loc is not None:
if loc.startswith("keep:"):
- colls.append(loc[5:])
+ locations.append(loc[5:])
+
+ docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None)
+ if docker_image is not None:
+ ds = docker_image.split(":", 1)
+ tag = ds[1] if len(ds)==2 else 'latest'
+ docker_images[ds[0]] = tag
+
for x in obj:
for x in obj:
- workflow_collections(obj[x],
coll
s)
- if isinstance(obj, list):
+ workflow_collections(obj[x],
locations, docker_image
s)
+
el
if isinstance(obj, list):
for x in obj:
for x in obj:
- workflow_collections(x,
coll
s)
+ workflow_collections(x,
locations, docker_image
s)
# copy_collections(obj, src, dst, args)
#
# copy_collections(obj, src, dst, args)
#
@@
-539,7
+561,7
@@
def migrate_jobspec(jobspec, src, dst, dst_repo, args):
# names. The return value is undefined.
#
def copy_git_repos(p, src, dst, dst_repo, args):
# names. The return value is undefined.
#
def copy_git_repos(p, src, dst, dst_repo, args):
- for component in p['components'].
iter
values():
+ for component in p['components'].values():
migrate_jobspec(component, src, dst, dst_repo, args)
if 'job' in component:
migrate_jobspec(component['job'], src, dst, dst_repo, args)
migrate_jobspec(component, src, dst, dst_repo, args)
if 'job' in component:
migrate_jobspec(component['job'], src, dst, dst_repo, args)
@@
-760,8
+782,8
@@
def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_
git_url = None
for url in priority:
if url.startswith("http"):
git_url = None
for url in priority:
if url.startswith("http"):
- u = urlparse.urlsplit(url)
- baseurl = urlparse.urlunsplit((u.scheme, u.netloc, "", "", ""))
+ u = url
lib.
parse.urlsplit(url)
+ baseurl = url
lib.
parse.urlunsplit((u.scheme, u.netloc, "", "", ""))
git_config = ["-c", "credential.%s/.username=none" % baseurl,
"-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
else:
git_config = ["-c", "credential.%s/.username=none" % baseurl,
"-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
else:
@@
-785,7
+807,7
@@
def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_
if git_url.startswith("http:"):
if allow_insecure_http:
if git_url.startswith("http:"):
if allow_insecure_http:
- logger.warn("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
+ logger.warn
ing
("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
else:
raise Exception("Refusing to use insecure git url %s, use %s if you really want this." % (git_url, allow_insecure_http_opt))
else:
raise Exception("Refusing to use insecure git url %s, use %s if you really want this." % (git_url, allow_insecure_http_opt))
@@
-845,7
+867,7
@@
def copy_docker_images(pipeline, src, dst, args):
runtime_constraints field from src to dst."""
logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
runtime_constraints field from src to dst."""
logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
- for c_name, c_info in pipeline['components'].ite
rite
ms():
+ for c_name, c_info in pipeline['components'].items():
if ('runtime_constraints' in c_info and
'docker_image' in c_info['runtime_constraints']):
copy_docker_image(
if ('runtime_constraints' in c_info and
'docker_image' in c_info['runtime_constraints']):
copy_docker_image(