import json
import os
import re
-import subprocess32 as subprocess
import sys
import tarfile
import tempfile
import shutil
import _strptime
import fcntl
-
from operator import itemgetter
from stat import *
+if os.name == "posix" and sys.version_info[0] < 3:
+ import subprocess32 as subprocess
+else:
+ import subprocess
+
import arvados
import arvados.util
import arvados.commands._util as arv_cmd
'--no-pull', action='store_false', dest='pull',
help="Use locally installed image only, don't pull image from Docker registry (default)")
-keepdocker_parser.add_argument(
- 'image', nargs='?',
- help="Docker image to upload: repo, repo:tag, or hash")
-keepdocker_parser.add_argument(
- 'tag', nargs='?',
- help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name")
-
# Combine keepdocker options listed above with run_opts options of arv-put.
# The options inherited from arv-put include --name, --project-uuid,
# --progress/--no-progress/--batch-progress and --resume/--no-resume.
description="Upload or list Docker images in Arvados",
parents=[keepdocker_parser, arv_put.run_opts, arv_cmd.retry_opt])
+arg_parser.add_argument(
+ 'image', nargs='?',
+ help="Docker image to upload: repo, repo:tag, or hash")
+arg_parser.add_argument(
+ 'tag', nargs='?',
+ help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name")
+
class DockerError(Exception):
pass
cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash],
stdout=subprocess.PIPE)
try:
- image_id = next(cmd.stdout).decode().strip()
+ image_id = next(cmd.stdout).decode('utf-8').strip()
if image_id.startswith('sha256:'):
return 'v2'
elif ':' not in image_id:
next(list_output) # Ignore the header line
for line in list_output:
words = line.split()
- words = [word.decode() for word in words]
+ words = [word.decode('utf-8') for word in words]
size_index = len(words) - 2
repo, tag, imageid = words[:3]
ctime = ' '.join(words[3:size_index])
Docker metadata links to sort them from least to most preferred.
"""
try:
- image_timestamp = ciso8601.parse_datetime_unaware(
+ image_timestamp = ciso8601.parse_datetime_as_naive(
link['properties']['image_timestamp'])
except (KeyError, ValueError):
image_timestamp = EARLIEST_DATETIME
- return (image_timestamp,
- ciso8601.parse_datetime_unaware(link['created_at']))
+ try:
+ created_timestamp = ciso8601.parse_datetime_as_naive(link['created_at'])
+ except ValueError:
+ created_timestamp = None
+ return (image_timestamp, created_timestamp)
def _get_docker_links(api_client, num_retries, **kwargs):
links = arvados.util.list_all(api_client.links().list,
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- logger.error(error.message)
+ logger.error(str(error))
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.name is None:
if image_repo_tag:
- collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12])
+ collection_name = 'Docker image {} {}'.format(image_repo_tag.replace("/", " "), image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
coll_uuid = api.collections().create(
body={"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
- "owner_uuid": parent_project_uuid},
+ "owner_uuid": parent_project_uuid,
+ "properties": {"docker-image-repo-tag": image_repo_tag}},
ensure_unique_name=True
).execute(num_retries=args.retries)['uuid']
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
+ if arguments is None:
+ arguments = sys.argv[1:]
+ arguments = [i for i in arguments if i not in (args.image, args.tag, image_repo_tag)]
put_args = keepdocker_parser.parse_known_args(arguments)[1]
+ # Don't fail when cached manifest is invalid, just ignore the cache.
+ put_args += ['--batch']
+
if args.name is None:
put_args += ['--name', collection_name]
put_args + ['--filename', outfile_name, image_file.name], stdout=stdout,
install_sig_handlers=install_sig_handlers).strip()
+ # Managed properties could be already set
+ coll_properties = api.collections().get(uuid=coll_uuid).execute(num_retries=args.retries).get('properties', {})
+ coll_properties.update({"docker-image-repo-tag": image_repo_tag})
+
+ api.collections().update(uuid=coll_uuid, body={"properties": coll_properties}).execute(num_retries=args.retries)
+
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
- image_metadata = json.load(json_file)
+ image_metadata = json.loads(json_file.read().decode('utf-8'))
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}