tools/docker-migrator/migrate.py

   1 #!/usr/bin/env python
   2
   3 import arvados
   4 import arvados.util
   5 from arvados.collection import CollectionReader
   6 import arvados.commands.keepdocker
   7 import re
   8 import subprocess
   9 import os
  10 import tempfile
  11 import shutil
  12
  13 from pprint import pprint
  14
  15 def main():
  16     api_client  = arvados.api()
  17
  18     images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
  19
  20     is_new = lambda img: img['dockerhash'].startswith('sha256:')
  21
  22     count_new = 0
  23     old_images = []
  24     for uuid, img in images:
  25         if img["dockerhash"].startswith("sha256:"):
  26             continue
  27         key = (img["repo"], img["tag"], img["timestamp"])
  28         old_images.append(img)
  29
  30     migration_links = arvados.util.list_all(api_client.links().list, filters=[
  31         ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
  32         ['name', '=', arvados.commands.keepdocker._migration_link_name],
  33     ])
  34
  35     already_migrated = set()
  36     for m in migration_links:
  37         already_migrated.add(m["tail_uuid"])
  38
  39     need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
  40
  41     print "Already migrated %i images" % (len(already_migrated))
  42     print "Need to migrate %i images" % (len(need_migrate))
  43
  44     for old_image in need_migrate:
  45         print "Migrating %s" % (old_image["collection"])
  46
  47         col = CollectionReader(old_image["collection"])
  48         tarfile = col.keys()[0]
  49
  50         try:
  51             varlibdocker = tempfile.mkdtemp()
  52             with tempfile.NamedTemporaryFile() as envfile:
  53                 envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
  54                 envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
  55                 envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
  56                 envfile.flush()
  57
  58                 dockercmd = ["docker", "run",
  59                              "--privileged",
  60                              "--rm",
  61                              "--env-file", envfile.name,
  62                              "--volume", "%s:/var/lib/docker" % varlibdocker,
  63                              "arvados/docker19-migrate",
  64                              "/root/migrate.sh",
  65                              "%s/%s" % (old_image["collection"], tarfile),
  66                              tarfile[0:40],
  67                              old_image["repo"],
  68                              old_image["tag"],
  69                              col.api_response()["owner_uuid"]]
  70
  71                 out = subprocess.check_output(dockercmd)
  72
  73             new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
  74             api_client.links().create(body={"link": {
  75                 'owner_uuid': col.api_response()["owner_uuid"],
  76                 'link_class': arvados.commands.keepdocker._migration_link_class,
  77                 'name': arvados.commands.keepdocker._migration_link_name,
  78                 'tail_uuid': old_image["collection"],
  79                 'head_uuid': new_collection.group(1)
  80                 }}).execute(num_retries=3)
  81
  82             print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
  83         finally:
  84             shutil.rmtree(varlibdocker)
  85
  86     print "All done"
  87
  88
  89 main()