From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Tue, 21 Mar 2017 18:02:09 +0000 (-0400)
Subject: 8567: Use vfs Docker graph driver for maximum compatability.  --dry-run reports
X-Git-Tag: 1.1.0~357^2~2
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/bbe8547e183a66fd551adffe59fbd4f5c146677c

8567: Use vfs Docker graph driver for maximum compatability.  --dry-run reports
maximum image size.  Add --tempdir option.
---

diff --git a/docker/migrate-docker19/dnd.sh b/docker/migrate-docker19/dnd.sh
index ce72601bff..ec6f1e3e12 100755
--- a/docker/migrate-docker19/dnd.sh
+++ b/docker/migrate-docker19/dnd.sh
@@ -14,7 +14,7 @@ CGROUP=/sys/fs/cgroup
 [ -d $CGROUP ] || mkdir $CGROUP
 
 if mountpoint -q $CGROUP ; then
-    break
+    true
 else
     mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
 fi
@@ -52,10 +52,10 @@ do
         # Systemd and OpenRC (and possibly others) both create such a
         # cgroup. To avoid the aforementioned bug, we symlink "foo" to
         # "name=foo". This shouldn't have any adverse effect.
-        echo $SUBSYS | grep -q ^name= && {
-                NAME=$(echo $SUBSYS | sed s/^name=//)
-                ln -s $SUBSYS $CGROUP/$NAME
-        }
+        #echo $SUBSYS | grep -q ^name= && {
+        #        NAME=$(echo $SUBSYS | sed s/^name=//)
+        #        ln -s $SUBSYS $CGROUP/$NAME
+        #}
 
         # Likewise, on at least one system, it has been reported that
         # systemd would mount the CPU and CPU accounting controllers
@@ -96,6 +96,4 @@ rm -rf /var/run/docker.pid
 
 read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
 
-if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
-    docker daemon $DOCKER_DAEMON_ARGS
-fi
+exec docker daemon --storage-driver=vfs $DOCKER_DAEMON_ARGS
diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index e75095bf6c..3705a9538c 100644
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -64,11 +64,16 @@ def main(arguments=None):
         '--print-unmigrated', action='store_true',
         default=False, help="Print list of images needing migration.")
 
+    migrate19_parser.add_argument('--tempdir', help="Set temporary directory")
+
     migrate19_parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
                                   default=None, help="List of images to be migrated")
 
     args = migrate19_parser.parse_args(arguments)
 
+    if args.tempdir:
+        tempfile.tempdir = args.tempdir
+
     only_migrate = None
     if args.infile:
         only_migrate = set()
@@ -105,14 +110,19 @@ def main(arguments=None):
 
     items = arvados.util.list_all(api_client.collections().list,
                                   filters=[["uuid", "in", [img["collection"] for img in old_images]]],
-                                  select=["uuid", "portable_data_hash"])
-    uuid_to_pdh = {i["uuid"]: i["portable_data_hash"] for i in items}
+                                  select=["uuid", "portable_data_hash", "manifest_text", "owner_uuid"])
+    uuid_to_collection = {i["uuid"]: i for i in items}
 
     need_migrate = {}
+    biggest = 0
     for img in old_images:
-        pdh = uuid_to_pdh[img["collection"]]
+        i = uuid_to_collection[img["collection"]]
+        pdh = i["portable_data_hash"]
         if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
             need_migrate[pdh] = img
+            with CollectionReader(i["manifest_text"]) as c:
+                if c.values()[0].size() > biggest:
+                    biggest = c.values()[0].size()
 
     if args.print_unmigrated:
         only_migrate = set()
@@ -122,6 +132,8 @@ def main(arguments=None):
 
     logger.info("Already migrated %i images", len(already_migrated))
     logger.info("Need to migrate %i images", len(need_migrate))
+    logger.info("Using tempdir %s", tempfile.gettempdir())
+    logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20))
 
     if args.dry_run:
         return
@@ -130,16 +142,17 @@ def main(arguments=None):
     failures = []
     count = 1
     for old_image in need_migrate.values():
-        if uuid_to_pdh[old_image["collection"]] in already_migrated:
+        if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
             continue
 
-        logger.info("[%i/%i] Migrating %s:%s (%s)", count, len(need_migrate), old_image["repo"], old_image["tag"], old_image["collection"])
+        oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
+        tarfile = oldcol.keys()[0]
+
+        logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
+                    old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
         count += 1
         start = time.time()
 
-        oldcol = CollectionReader(old_image["collection"])
-        tarfile = oldcol.keys()[0]
-
         varlibdocker = tempfile.mkdtemp()
         try:
             with tempfile.NamedTemporaryFile() as envfile:
@@ -160,7 +173,7 @@ def main(arguments=None):
                              tarfile[0:40],
                              old_image["repo"],
                              old_image["tag"],
-                             oldcol.api_response()["owner_uuid"]]
+                             uuid_to_collection[old_image["collection"]]["owner_uuid"]]
 
                 proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 out, err = proc.communicate()