10111: Merge branch 'master' into 10111-cr-provenance-graph

[arvados.git] / sdk / python / arvados / commands / keepdocker.py

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 5f7749befa3c5b04a8075f0bdae85dc67b16122e..57832483236fe5e404e9c64b7d3158336e88cd2d 100644 (file)
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -6,10 +6,12 @@ import datetime
 import errno
 import json
 import os
+import re
 import subprocess
 import sys
 import tarfile
 import tempfile
+import shutil
 import _strptime
 
 from operator import itemgetter
@@ -19,10 +21,17 @@ import arvados
 import arvados.util
 import arvados.commands._util as arv_cmd
 import arvados.commands.put as arv_put
+from arvados.collection import CollectionReader
 import ciso8601
+import logging
+import arvados.config
 
 from arvados._version import __version__
 
+logger = logging.getLogger('arvados.keepdocker')
+logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG')
+                else logging.INFO)
+
 EARLIEST_DATETIME = datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0)
 STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
 
@@ -102,15 +111,15 @@ def docker_image_format(image_hash):
 def docker_image_compatible(api, image_hash):
     supported = api._rootDesc.get('dockerImageFormats', [])
     if not supported:
-        print >>sys.stderr, "arv-keepdocker: warning: server does not specify supported image formats (see docker_image_formats in server config). Continuing."
-        return True
+        logger.warn("server does not specify supported image formats (see docker_image_formats in server config).")
+        return False
 
     fmt = docker_image_format(image_hash)
     if fmt in supported:
         return True
     else:
-        print >>sys.stderr, "arv-keepdocker: image format is {!r} " \
-            "but server supports only {!r}".format(fmt, supported)
+        logger.error("image format is {!r} " \
+            "but server supports only {!r}".format(fmt, supported))
         return False
 
 def docker_images():
@@ -323,6 +332,12 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
 def items_owned_by(owner_uuid, arv_items):
     return (item for item in arv_items if item['owner_uuid'] == owner_uuid)
 
+def _uuid2pdh(api, uuid):
+    return api.collections().list(
+        filters=[['uuid', '=', uuid]],
+        select=['portable_data_hash'],
+    ).execute()['items'][0]['portable_data_hash']
+
 def main(arguments=None, stdout=sys.stdout):
     args = arg_parser.parse_args(arguments)
     api = arvados.api('v1')
@@ -330,8 +345,14 @@ def main(arguments=None, stdout=sys.stdout):
     if args.image is None or args.image == 'images':
         fmt = "{:30}  {:10}  {:12}  {:29}  {:20}\n"
         stdout.write(fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED"))
-        for i, j in list_images_in_arv(api, args.retries):
-            stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
+        try:
+            for i, j in list_images_in_arv(api, args.retries):
+                stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
+        except IOError as e:
+            if e.errno == errno.EPIPE:
+                pass
+            else:
+                raise
         sys.exit(0)
 
     # Pull the image if requested, unless the image is specified as a hash
@@ -342,15 +363,15 @@ def main(arguments=None, stdout=sys.stdout):
     try:
         image_hash = find_one_image_hash(args.image, args.tag)
     except DockerError as error:
-        print >>sys.stderr, "arv-keepdocker:", error.message
+        logger.error(error.message)
         sys.exit(1)
 
     if not docker_image_compatible(api, image_hash):
         if args.force_image_format:
-            print >>sys.stderr, "arv-keepdocker: forcing incompatible image"
+            logger.warn("forcing incompatible image")
         else:
-            print >>sys.stderr, "arv-keepdocker: refusing to store " \
-                "incompatible format (use --force-image-format to override)"
+            logger.error("refusing to store " \
+                "incompatible format (use --force-image-format to override)")
             sys.exit(1)
 
     image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
@@ -392,7 +413,7 @@ def main(arguments=None, stdout=sys.stdout):
                         api, args.retries,
                         filters=[['link_class', '=', 'docker_image_repo+tag'],
                                  ['name', '=', image_repo_tag],
-                                 ['head_uuid', 'in', collections]])
+                                 ['head_uuid', 'in', [c["uuid"] for c in collections]]])
                 else:
                     existing_repo_tag = []