8567: Add status reporting to migrate script.

diff --git a/tools/docker-migrator/README b/tools/docker-migrator/README
new file mode 100644 (file)
index 0000000..d94eb88
--- /dev/null
+++ b/tools/docker-migrator/README
@@ -0,0 +1,24 @@
+Docker image format migration tool for Arvados.
+
+This converts Docker images stored in Arvados from image format v1
+(Docker <= 1.9) to image format v2 (Docker >= 1.10).
+
+Requires Docker running on the local host.
+
+Usage:
+
+1) Run ./build.sh to create arvados/docker19-migrate Docker image.
+
+2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
+
+3) Run ./migrate.py
+
+This will query Arvados for v1 format Docker images.  For each image that does
+not already have a corresponding v2 format image (as indicated by a
+docker_image_migration tag) it will perform the following process:
+
+i) download the image from Arvados
+ii) load it into Docker
+iii) update the Docker version, which updates the image
+iv) save the v2 format image and upload to Arvados
+v) create a migration link

diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 23b81668df235f06582b2483f35121517bf1e78d..ec10b75734a79d311cd7f30cb9c403d3e9037f22 100755 (executable)
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -36,31 +36,39 @@ def main():
     for m in migration_links:
         already_migrated.add(m["tail_uuid"])
 
-    for old_image in old_images:
-        if old_image["collection"] in already_migrated:
-            continue
+    need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+    print "Already migrated %i images" % (len(already_migrated))
+    print "Need to migrate %i images" % (len(need_migrate))
+
+    for old_image in need_migrate:
+        print "Migrating %s" % (old_image["collection"])
+
         col = CollectionReader(old_image["collection"])
         tarfile = col.keys()[0]
 
-        varlibdocker = tempfile.mkdtemp()
-
         try:
-            dockercmd = ["docker", "run",
-                         "--privileged",
-                         "--rm",
-                         "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
-                         "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
-                         "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
-                         "--volume", "%s:/var/lib/docker" % varlibdocker,
-                         "arvados/docker19-migrate",
-                         "/root/migrate.sh",
-                         "%s/%s" % (old_image["collection"], tarfile),
-                         tarfile[0:40],
-                         old_image["repo"],
-                         old_image["tag"],
-                         col.api_response()["owner_uuid"]]
-
-            out = subprocess.check_output(dockercmd)
+            varlibdocker = tempfile.mkdtemp()
+            with tempfile.NamedTemporaryFile() as envfile:
+                envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+                envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+                envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+                envfile.flush()
+
+                dockercmd = ["docker", "run",
+                             "--privileged",
+                             "--rm",
+                             "--env-file", envfile.name,
+                             "--volume", "%s:/var/lib/docker" % varlibdocker,
+                             "arvados/docker19-migrate",
+                             "/root/migrate.sh",
+                             "%s/%s" % (old_image["collection"], tarfile),
+                             tarfile[0:40],
+                             old_image["repo"],
+                             old_image["tag"],
+                             col.api_response()["owner_uuid"]]
+
+                out = subprocess.check_output(dockercmd)
 
             new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
             api_client.links().create(body={"link": {
@@ -75,5 +83,7 @@ def main():
         finally:
             shutil.rmtree(varlibdocker)
 
+    print "All done"
+
 
 main()