17800: Avoids saving a new empty collection.

[arvados.git] / sdk / python / arvados / commands / migrate19.py

diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index a9768a0374b18fe32e4611797fa4338137b0e5c1..3ce47b20660bc68c51833b981cdfdda17c6672e8 100644 (file)
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -1,3 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import print_function
+from __future__ import division
 import argparse
 import time
 import sys
@@ -125,17 +131,20 @@ def main(arguments=None):
     uuid_to_collection = {i["uuid"]: i for i in items}
 
     need_migrate = {}
+    totalbytes = 0
     biggest = 0
     biggest_pdh = None
     for img in old_images:
         i = uuid_to_collection[img["collection"]]
         pdh = i["portable_data_hash"]
-        if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
+        if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate):
             need_migrate[pdh] = img
             with CollectionReader(i["manifest_text"]) as c:
-                if c.values()[0].size() > biggest:
-                    biggest = c.values()[0].size()
+                size = list(c.values())[0].size()
+                if size > biggest:
+                    biggest = size
                     biggest_pdh = pdh
+                totalbytes += size
 
 
     if args.storage_driver == "vfs":
@@ -152,13 +161,14 @@ def main(arguments=None):
     logger.info("Already migrated %i images", len(already_migrated))
     logger.info("Need to migrate %i images", len(need_migrate))
     logger.info("Using tempdir %s", tempfile.gettempdir())
-    logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
+    logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest>>20)
+    logger.info("Total data to migrate about %i MiB", totalbytes>>20)
 
     df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
     ln = df_out.splitlines()[1]
     filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
     if int(available) <= will_need:
-        logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
+        logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)>>20, int(will_need)>>20)
         if not args.force:
             exit(1)
         else:
@@ -170,15 +180,15 @@ def main(arguments=None):
     success = []
     failures = []
     count = 1
-    for old_image in need_migrate.values():
+    for old_image in list(need_migrate.values()):
         if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
             continue
 
         oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
-        tarfile = oldcol.keys()[0]
+        tarfile = list(oldcol.keys())[0]
 
         logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
-                    old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
+                    old_image["tag"], old_image["collection"], list(oldcol.values())[0].size()>>20)
         count += 1
         start = time.time()
 
@@ -186,10 +196,10 @@ def main(arguments=None):
         dockercache = tempfile.mkdtemp()
         try:
             with tempfile.NamedTemporaryFile() as envfile:
-                envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
-                envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
-                if "ARVADOS_API_HOST_INSECURE" in os.environ:
-                    envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+                envfile.write("ARVADOS_API_HOST=%s\n" % (arvados.config.get("ARVADOS_API_HOST")))
+                envfile.write("ARVADOS_API_TOKEN=%s\n" % (arvados.config.get("ARVADOS_API_TOKEN")))
+                if arvados.config.get("ARVADOS_API_HOST_INSECURE"):
+                    envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (arvados.config.get("ARVADOS_API_HOST_INSECURE")))
                 envfile.flush()
 
                 dockercmd = ["docker", "run",
@@ -218,7 +228,7 @@ def main(arguments=None):
 
                 if initial_space:
                     isp = int(initial_space.group(1))
-                    logger.debug("Available space initially: %i MiB", (isp)/(2**20))
+                    logger.info("Available space initially: %i MiB", (isp)/(2**20))
                     if imgload_space:
                         sp = int(imgload_space.group(1))
                         logger.debug("Used after load: %i MiB", (isp-sp)/(2**20))
@@ -227,11 +237,11 @@ def main(arguments=None):
                         logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20))
                     if keepdocker_space:
                         sp = int(keepdocker_space.group(1))
-                        logger.debug("Used after upload: %i MiB", (isp-sp)/(2**20))
+                        logger.info("Used after upload: %i MiB", (isp-sp)/(2**20))
 
                 if cleanup_space:
                     sp = int(cleanup_space.group(1))
-                    logger.info("Available after cleanup: %i MiB", (sp)/(2**20))
+                    logger.debug("Available after cleanup: %i MiB", (sp)/(2**20))
 
                 if proc.returncode != 0:
                     logger.error("Failed with return code %i", proc.returncode)