X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c8de202a8ba2b7e128f9698c4b57f0d7d9e89c1d..e09e57378e5f98235e472a35c18a3e3a2bac97fb:/sdk/python/arvados/commands/keepdocker.py diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py index fa4e44136b..c96e2cbd60 100644 --- a/sdk/python/arvados/commands/keepdocker.py +++ b/sdk/python/arvados/commands/keepdocker.py @@ -22,19 +22,12 @@ STAT_CACHE_ERRORS = (IOError, OSError, ValueError) DockerImage = namedtuple('DockerImage', ['repo', 'tag', 'hash', 'created', 'vsize']) -opt_parser = argparse.ArgumentParser(add_help=False) -opt_parser.add_argument( +keepdocker_parser = argparse.ArgumentParser(add_help=False) +keepdocker_parser.add_argument( '-f', '--force', action='store_true', default=False, help="Re-upload the image even if it already exists on the server") -opt_parser.add_argument( - '--project-uuid', - help="Add the Docker image and metadata to the specified project. Goes into user 'home' project by default.") -opt_parser.add_argument( - '--name', - help="Name to use for the collection that will contain the docker image.") - -_group = opt_parser.add_mutually_exclusive_group() +_group = keepdocker_parser.add_mutually_exclusive_group() _group.add_argument( '--pull', action='store_true', default=False, help="Try to pull the latest image from Docker registry") @@ -42,16 +35,19 @@ _group.add_argument( '--no-pull', action='store_false', dest='pull', help="Use locally installed image only, don't pull image from Docker registry (default)") -opt_parser.add_argument( +keepdocker_parser.add_argument( 'image', nargs='?', help="Docker image to upload, as a repository name or hash") -opt_parser.add_argument( +keepdocker_parser.add_argument( 'tag', nargs='?', default='latest', help="Tag of the Docker image to upload (default 'latest')") +# Combine keepdocker options listed above with run_opts options of arv-put. +# The options inherited from arv-put include --name, --project-uuid, +# --progress/--no-progress/--batch-progress and --resume/--no-resume. arg_parser = argparse.ArgumentParser( description="Upload or list Docker images in Arvados", - parents=[opt_parser, arv_put.run_opts]) + parents=[keepdocker_parser, arv_put.run_opts]) class DockerError(Exception): pass @@ -193,10 +189,10 @@ def list_images_in_arv(): st = sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"])) - fmt = "{:30} {:10} {:12} {:38} {:20}" - print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "KEEP LOCATOR", "CREATED") + fmt = "{:30} {:10} {:12} {:29} {:20}" + print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED") for i, j in st: - print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:11], i, j["timestamp"].strftime("%c"))) + print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c"))) def main(arguments=None): args = arg_parser.parse_args(arguments) @@ -215,18 +211,71 @@ def main(arguments=None): except DockerError as error: print >>sys.stderr, "arv-keepdocker:", error.message sys.exit(1) + + image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None + + if args.name is None: + if image_repo_tag: + collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12]) + else: + collection_name = 'Docker image {}'.format(image_hash[0:12]) + else: + collection_name = args.name + + api = arvados.api('v1') + if not args.force: - # Abort if this image is already in Arvados. - existing_links = arvados.api('v1').links().list( + # Check if this image is already in Arvados. + + # Project where everything should be owned + parent_project_uuid = args.project_uuid if args.project_uuid else api.users().current().execute()['uuid'] + + # Find image hash tags + existing_links = api.links().list( filters=[['link_class', '=', 'docker_image_hash'], ['name', '=', image_hash]]).execute()['items'] if existing_links: - message = [ - "arv-keepdocker: Image {} already stored in collection(s):". - format(image_hash)] - message.extend(link['head_uuid'] for link in existing_links) - print >>sys.stderr, "\n".join(message) - sys.exit(0) + # get readable collections + collections = api.collections().list( + filters=[['uuid', 'in', [link['head_uuid'] for link in existing_links]]], + select=["uuid", "owner_uuid", "name", "manifest_text"]).execute()['items'] + + if collections: + # check for repo+tag links on these collections + existing_repo_tag = (api.links().list( + filters=[['link_class', '=', 'docker_image_repo+tag'], + ['name', '=', image_repo_tag], + ['head_uuid', 'in', collections]]).execute()['items']) if image_repo_tag else [] + + # Filter on elements owned by the parent project + owned_col = [c for c in collections if c['owner_uuid'] == parent_project_uuid] + owned_img = [c for c in existing_links if c['owner_uuid'] == parent_project_uuid] + owned_rep = [c for c in existing_repo_tag if c['owner_uuid'] == parent_project_uuid] + + if owned_col: + # already have a collection owned by this project + coll_uuid = owned_col[0]['uuid'] + else: + # create new collection owned by the project + coll_uuid = api.collections().create(body={"manifest_text": collections[0]['manifest_text'], + "name": collection_name, + "owner_uuid": parent_project_uuid}, + ensure_unique_name=True).execute()['uuid'] + + link_base = {'owner_uuid': parent_project_uuid, + 'head_uuid': coll_uuid } + + if not owned_img: + # create image link owned by the project + make_link('docker_image_hash', image_hash, **link_base) + + if not owned_rep and image_repo_tag: + # create repo+tag link owned by the project + make_link('docker_image_repo+tag', image_repo_tag, **link_base) + + print(coll_uuid) + + sys.exit(0) # Open a file for the saved image, and write it if needed. outfile_name = '{}.tar'.format(image_hash) @@ -236,15 +285,10 @@ def main(arguments=None): # Call arv-put with switches we inherited from it # (a.k.a., switches that aren't our own). - put_args = opt_parser.parse_known_args(arguments)[1] + put_args = keepdocker_parser.parse_known_args(arguments)[1] if args.name is None: - put_args += ['--name', 'Docker image {}:{} {}'.format(args.image, args.tag, image_hash[0:11])] - else: - put_args += ['--name', args.name] - - if args.project_uuid is not None: - put_args += ['--project-uuid', args.project_uuid] + put_args += ['--name', collection_name] coll_uuid = arv_put.main( put_args + ['--filename', outfile_name, image_file.name]).strip() @@ -263,8 +307,8 @@ def main(arguments=None): link_base['owner_uuid'] = args.project_uuid make_link('docker_image_hash', image_hash, **link_base) - if not image_hash.startswith(args.image.lower()): - make_link('docker_image_repo+tag', '{}:{}'.format(args.image, args.tag), + if image_repo_tag: + make_link('docker_image_repo+tag', image_repo_tag, **link_base) # Clean up.