+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import print_function
+from __future__ import division
import argparse
import time
import sys
uuid_to_collection = {i["uuid"]: i for i in items}
need_migrate = {}
+ totalbytes = 0
biggest = 0
biggest_pdh = None
for img in old_images:
i = uuid_to_collection[img["collection"]]
pdh = i["portable_data_hash"]
- if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
+ if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate):
need_migrate[pdh] = img
with CollectionReader(i["manifest_text"]) as c:
- if c.values()[0].size() > biggest:
- biggest = c.values()[0].size()
+ size = list(c.values())[0].size()
+ if size > biggest:
+ biggest = size
biggest_pdh = pdh
+ totalbytes += size
if args.storage_driver == "vfs":
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
+ logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest>>20)
+ logger.info("Total data to migrate about %i MiB", totalbytes>>20)
df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
ln = df_out.splitlines()[1]
filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
if int(available) <= will_need:
- logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
+ logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)>>20, int(will_need)>>20)
if not args.force:
exit(1)
else:
success = []
failures = []
count = 1
- for old_image in need_migrate.values():
+ for old_image in list(need_migrate.values()):
if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
continue
oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
- tarfile = oldcol.keys()[0]
+ tarfile = list(oldcol.keys())[0]
logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
- old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
+ old_image["tag"], old_image["collection"], list(oldcol.values())[0].size()>>20)
count += 1
start = time.time()