8567: Improve disk space accounting. Default to use overlay storage driver (but...

[arvados.git] / sdk / python / arvados / commands / migrate19.py

diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index 4be99961ba0febcfcf36b254f582945427b133e8..a9768a0374b18fe32e4611797fa4338137b0e5c1 100644 (file)
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -56,6 +56,14 @@ def main(arguments=None):
     migrate19_parser.add_argument(
         '--version', action='version', version="%s %s" % (sys.argv[0], __version__),
         help='Print version and exit.')
+    migrate19_parser.add_argument(
+        '--verbose', action="store_true", help="Print stdout/stderr even on success")
+    migrate19_parser.add_argument(
+        '--force', action="store_true", help="Try to migrate even if there isn't enough space")
+
+    migrate19_parser.add_argument(
+        '--storage-driver', type=str, default="overlay",
+        help="Docker storage driver, e.g. aufs, overlay, vfs")
 
     exgroup = migrate19_parser.add_mutually_exclusive_group()
     exgroup.add_argument(
@@ -74,6 +82,9 @@ def main(arguments=None):
     if args.tempdir:
         tempfile.tempdir = args.tempdir
 
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
     only_migrate = None
     if args.infile:
         only_migrate = set()
@@ -115,6 +126,7 @@ def main(arguments=None):
 
     need_migrate = {}
     biggest = 0
+    biggest_pdh = None
     for img in old_images:
         i = uuid_to_collection[img["collection"]]
         pdh = i["portable_data_hash"]
@@ -123,17 +135,34 @@ def main(arguments=None):
             with CollectionReader(i["manifest_text"]) as c:
                 if c.values()[0].size() > biggest:
                     biggest = c.values()[0].size()
+                    biggest_pdh = pdh
+
+
+    if args.storage_driver == "vfs":
+        will_need = (biggest*20)
+    else:
+        will_need = (biggest*2.5)
 
     if args.print_unmigrated:
         only_migrate = set()
         for pdh in need_migrate:
-            print pdh
+            print(pdh)
         return
 
     logger.info("Already migrated %i images", len(already_migrated))
     logger.info("Need to migrate %i images", len(need_migrate))
     logger.info("Using tempdir %s", tempfile.gettempdir())
-    logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20))
+    logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
+
+    df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
+    ln = df_out.splitlines()[1]
+    filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
+    if int(available) <= will_need:
+        logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
+        if not args.force:
+            exit(1)
+        else:
+            logger.warn("--force provided, will migrate anyway")
 
     if args.dry_run:
         return
@@ -169,23 +198,51 @@ def main(arguments=None):
                              "--env-file", envfile.name,
                              "--volume", "%s:/var/lib/docker" % varlibdocker,
                              "--volume", "%s:/root/.cache/arvados/docker" % dockercache,
-                             "arvados/migrate-docker19",
+                             "arvados/migrate-docker19:1.0",
                              "/root/migrate.sh",
                              "%s/%s" % (old_image["collection"], tarfile),
                              tarfile[0:40],
                              old_image["repo"],
                              old_image["tag"],
-                             uuid_to_collection[old_image["collection"]]["owner_uuid"]]
+                             uuid_to_collection[old_image["collection"]]["owner_uuid"],
+                             args.storage_driver]
 
                 proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 out, err = proc.communicate()
 
+                initial_space = re.search(r"Initial available space is (\d+)", out)
+                imgload_space = re.search(r"Available space after image load is (\d+)", out)
+                imgupgrade_space = re.search(r"Available space after image upgrade is (\d+)", out)
+                keepdocker_space = re.search(r"Available space after arv-keepdocker is (\d+)", out)
+                cleanup_space = re.search(r"Available space after cleanup is (\d+)", out)
+
+                if initial_space:
+                    isp = int(initial_space.group(1))
+                    logger.debug("Available space initially: %i MiB", (isp)/(2**20))
+                    if imgload_space:
+                        sp = int(imgload_space.group(1))
+                        logger.debug("Used after load: %i MiB", (isp-sp)/(2**20))
+                    if imgupgrade_space:
+                        sp = int(imgupgrade_space.group(1))
+                        logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20))
+                    if keepdocker_space:
+                        sp = int(keepdocker_space.group(1))
+                        logger.debug("Used after upload: %i MiB", (isp-sp)/(2**20))
+
+                if cleanup_space:
+                    sp = int(cleanup_space.group(1))
+                    logger.info("Available after cleanup: %i MiB", (sp)/(2**20))
+
                 if proc.returncode != 0:
                     logger.error("Failed with return code %i", proc.returncode)
                     logger.error("--- Stdout ---\n%s", out)
                     logger.error("--- Stderr ---\n%s", err)
                     raise MigrationFailed()
 
+                if args.verbose:
+                    logger.info("--- Stdout ---\n%s", out)
+                    logger.info("--- Stderr ---\n%s", err)
+
             migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
             if migrated:
                 newcol = CollectionReader(migrated.group(1))