Merge branch '16265-security-updates' into dependabot/bundler/apps/workbench/loofah...

[arvados.git] / sdk / python / arvados / commands / keepdocker.py

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 55fc6b626d5de9798b989d2b252d158ca89baab6..6673888ab5e3f98a39c46df8baf673c5a610df76 100644 (file)
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -10,17 +10,20 @@ import errno
 import json
 import os
 import re
-import subprocess32 as subprocess
 import sys
 import tarfile
 import tempfile
 import shutil
 import _strptime
 import fcntl
-
 from operator import itemgetter
 from stat import *
 
+if os.name == "posix" and sys.version_info[0] < 3:
+    import subprocess32 as subprocess
+else:
+    import subprocess
+
 import arvados
 import arvados.util
 import arvados.commands._util as arv_cmd
@@ -61,13 +64,6 @@ _group.add_argument(
     '--no-pull', action='store_false', dest='pull',
     help="Use locally installed image only, don't pull image from Docker registry (default)")
 
-keepdocker_parser.add_argument(
-    'image', nargs='?',
-    help="Docker image to upload: repo, repo:tag, or hash")
-keepdocker_parser.add_argument(
-    'tag', nargs='?',
-    help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name")
-
 # Combine keepdocker options listed above with run_opts options of arv-put.
 # The options inherited from arv-put include --name, --project-uuid,
 # --progress/--no-progress/--batch-progress and --resume/--no-resume.
@@ -75,6 +71,13 @@ arg_parser = argparse.ArgumentParser(
         description="Upload or list Docker images in Arvados",
         parents=[keepdocker_parser, arv_put.run_opts, arv_cmd.retry_opt])
 
+arg_parser.add_argument(
+    'image', nargs='?',
+    help="Docker image to upload: repo, repo:tag, or hash")
+arg_parser.add_argument(
+    'tag', nargs='?',
+    help="Tag of the Docker image to upload (default 'latest'), if image is given as an untagged repo name")
+
 class DockerError(Exception):
     pass
 
@@ -102,7 +105,7 @@ def docker_image_format(image_hash):
     cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash],
                         stdout=subprocess.PIPE)
     try:
-        image_id = next(cmd.stdout).decode().strip()
+        image_id = next(cmd.stdout).decode('utf-8').strip()
         if image_id.startswith('sha256:'):
             return 'v2'
         elif ':' not in image_id:
@@ -133,6 +136,7 @@ def docker_images():
     next(list_output)  # Ignore the header line
     for line in list_output:
         words = line.split()
+        words = [word.decode('utf-8') for word in words]
         size_index = len(words) - 2
         repo, tag, imageid = words[:3]
         ctime = ' '.join(words[3:size_index])
@@ -226,12 +230,15 @@ def docker_link_sort_key(link):
     Docker metadata links to sort them from least to most preferred.
     """
     try:
-        image_timestamp = ciso8601.parse_datetime_unaware(
+        image_timestamp = ciso8601.parse_datetime_as_naive(
             link['properties']['image_timestamp'])
     except (KeyError, ValueError):
         image_timestamp = EARLIEST_DATETIME
-    return (image_timestamp,
-            ciso8601.parse_datetime_unaware(link['created_at']))
+    try:
+        created_timestamp = ciso8601.parse_datetime_as_naive(link['created_at'])
+    except ValueError:
+        created_timestamp = None
+    return (image_timestamp, created_timestamp)
 
 def _get_docker_links(api_client, num_retries, **kwargs):
     links = arvados.util.list_all(api_client.links().list,
@@ -383,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
     try:
         image_hash = find_one_image_hash(args.image, args.tag)
     except DockerError as error:
-        logger.error(error.message)
+        logger.error(str(error))
         sys.exit(1)
 
     if not docker_image_compatible(api, image_hash):
@@ -398,7 +405,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
 
     if args.name is None:
         if image_repo_tag:
-            collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12])
+            collection_name = 'Docker image {} {}'.format(image_repo_tag.replace("/", " "), image_hash[0:12])
         else:
             collection_name = 'Docker image {}'.format(image_hash[0:12])
     else:
@@ -456,7 +463,8 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
                         coll_uuid = api.collections().create(
                             body={"manifest_text": collections[0]['manifest_text'],
                                   "name": collection_name,
-                                  "owner_uuid": parent_project_uuid},
+                                  "owner_uuid": parent_project_uuid,
+                                  "properties": {"docker-image-repo-tag": image_repo_tag}},
                             ensure_unique_name=True
                             ).execute(num_retries=args.retries)['uuid']
 
@@ -485,6 +493,9 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
 
         # Call arv-put with switches we inherited from it
         # (a.k.a., switches that aren't our own).
+        if arguments is None:
+            arguments = sys.argv[1:]
+        arguments = [i for i in arguments if i not in (args.image, args.tag, image_repo_tag)]
         put_args = keepdocker_parser.parse_known_args(arguments)[1]
 
         if args.name is None:
@@ -494,6 +505,8 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
             put_args + ['--filename', outfile_name, image_file.name], stdout=stdout,
             install_sig_handlers=install_sig_handlers).strip()
 
+        api.collections().update(uuid=coll_uuid, body={"properties": {"docker-image-repo-tag": image_repo_tag}}).execute(num_retries=args.retries)
+
         # Read the image metadata and make Arvados links from it.
         image_file.seek(0)
         image_tar = tarfile.open(fileobj=image_file)
@@ -503,7 +516,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
         else:
             json_filename = raw_image_hash + '/json'
         json_file = image_tar.extractfile(image_tar.getmember(json_filename))
-        image_metadata = json.load(json_file)
+        image_metadata = json.loads(json_file.read().decode('utf-8'))
         json_file.close()
         image_tar.close()
         link_base = {'head_uuid': coll_uuid, 'properties': {}}