X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2957b917aaaefc1485e1c5293d413f0931b3030a..2f66d4cc05e9442a9bb69969744d0750a02a1ed4:/sdk/python/arvados/commands/keepdocker.py diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py index fe6beab510..6673888ab5 100644 --- a/sdk/python/arvados/commands/keepdocker.py +++ b/sdk/python/arvados/commands/keepdocker.py @@ -10,17 +10,20 @@ import errno import json import os import re -import subprocess import sys import tarfile import tempfile import shutil import _strptime import fcntl - from operator import itemgetter from stat import * +if os.name == "posix" and sys.version_info[0] < 3: + import subprocess32 as subprocess +else: + import subprocess + import arvados import arvados.util import arvados.commands._util as arv_cmd @@ -61,13 +64,6 @@ _group.add_argument( '--no-pull', action='store_false', dest='pull', help="Use locally installed image only, don't pull image from Docker registry (default)") -keepdocker_parser.add_argument( - 'image', nargs='?', - help="Docker image to upload: repo, repo:tag, or hash") -keepdocker_parser.add_argument( - 'tag', nargs='?', - help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name") - # Combine keepdocker options listed above with run_opts options of arv-put. # The options inherited from arv-put include --name, --project-uuid, # --progress/--no-progress/--batch-progress and --resume/--no-resume. @@ -75,6 +71,13 @@ arg_parser = argparse.ArgumentParser( description="Upload or list Docker images in Arvados", parents=[keepdocker_parser, arv_put.run_opts, arv_cmd.retry_opt]) +arg_parser.add_argument( + 'image', nargs='?', + help="Docker image to upload: repo, repo:tag, or hash") +arg_parser.add_argument( + 'tag', nargs='?', + help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name") + class DockerError(Exception): pass @@ -102,7 +105,7 @@ def docker_image_format(image_hash): cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash], stdout=subprocess.PIPE) try: - image_id = next(cmd.stdout).decode().strip() + image_id = next(cmd.stdout).decode('utf-8').strip() if image_id.startswith('sha256:'): return 'v2' elif ':' not in image_id: @@ -133,6 +136,7 @@ def docker_images(): next(list_output) # Ignore the header line for line in list_output: words = line.split() + words = [word.decode('utf-8') for word in words] size_index = len(words) - 2 repo, tag, imageid = words[:3] ctime = ' '.join(words[3:size_index]) @@ -226,12 +230,15 @@ def docker_link_sort_key(link): Docker metadata links to sort them from least to most preferred. """ try: - image_timestamp = ciso8601.parse_datetime_unaware( + image_timestamp = ciso8601.parse_datetime_as_naive( link['properties']['image_timestamp']) except (KeyError, ValueError): image_timestamp = EARLIEST_DATETIME - return (image_timestamp, - ciso8601.parse_datetime_unaware(link['created_at'])) + try: + created_timestamp = ciso8601.parse_datetime_as_naive(link['created_at']) + except ValueError: + created_timestamp = None + return (image_timestamp, created_timestamp) def _get_docker_links(api_client, num_retries, **kwargs): links = arvados.util.list_all(api_client.links().list, @@ -383,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) try: image_hash = find_one_image_hash(args.image, args.tag) except DockerError as error: - logger.error(error.message) + logger.error(str(error)) sys.exit(1) if not docker_image_compatible(api, image_hash): @@ -398,7 +405,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) if args.name is None: if image_repo_tag: - collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12]) + collection_name = 'Docker image {} {}'.format(image_repo_tag.replace("/", " "), image_hash[0:12]) else: collection_name = 'Docker image {}'.format(image_hash[0:12]) else: @@ -456,7 +463,8 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) coll_uuid = api.collections().create( body={"manifest_text": collections[0]['manifest_text'], "name": collection_name, - "owner_uuid": parent_project_uuid}, + "owner_uuid": parent_project_uuid, + "properties": {"docker-image-repo-tag": image_repo_tag}}, ensure_unique_name=True ).execute(num_retries=args.retries)['uuid'] @@ -485,6 +493,9 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) # Call arv-put with switches we inherited from it # (a.k.a., switches that aren't our own). + if arguments is None: + arguments = sys.argv[1:] + arguments = [i for i in arguments if i not in (args.image, args.tag, image_repo_tag)] put_args = keepdocker_parser.parse_known_args(arguments)[1] if args.name is None: @@ -494,6 +505,8 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) put_args + ['--filename', outfile_name, image_file.name], stdout=stdout, install_sig_handlers=install_sig_handlers).strip() + api.collections().update(uuid=coll_uuid, body={"properties": {"docker-image-repo-tag": image_repo_tag}}).execute(num_retries=args.retries) + # Read the image metadata and make Arvados links from it. image_file.seek(0) image_tar = tarfile.open(fileobj=image_file) @@ -503,7 +516,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) else: json_filename = raw_image_hash + '/json' json_file = image_tar.extractfile(image_tar.getmember(json_filename)) - image_metadata = json.load(json_file) + image_metadata = json.loads(json_file.read().decode('utf-8')) json_file.close() image_tar.close() link_base = {'head_uuid': coll_uuid, 'properties': {}}