+ This will query Arvados for v1 format Docker images. For each image that
+ does not already have a corresponding v2 format image (as indicated by a
+ docker_image_migration tag) it will perform the following process:
+
+ i) download the image from Arvados
+ ii) load it into Docker
+ iii) update the Docker version, which updates the image
+ iv) save the v2 format image and upload to Arvados
+ v) create a migration link
+
+ """
+
+ api_client = arvados.api()
+
+ user = api_client.users().current().execute()
+ if not user['is_admin']:
+ raise Exception("This command requires an admin token")
+ sys_uuid = user['uuid'][:12] + '000000000000000'
+
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
+ is_new = lambda img: img['dockerhash'].startswith('sha256:')
+
+ count_new = 0
+ old_images = []
+ for uuid, img in images:
+ if img["dockerhash"].startswith("sha256:"):
+ continue
+ key = (img["repo"], img["tag"], img["timestamp"])
+ old_images.append(img)
+
+ migration_links = arvados.util.list_all(api_client.links().list, filters=[
+ ['link_class', '=', _migration_link_class],
+ ['name', '=', _migration_link_name],
+ ])
+
+ already_migrated = set()
+ for m in migration_links:
+ already_migrated.add(m["tail_uuid"])
+
+ items = arvados.util.list_all(api_client.collections().list,
+ filters=[["uuid", "in", [img["collection"] for img in old_images]]],
+ select=["uuid", "portable_data_hash"])
+ uuid_to_pdh = {i["uuid"]: i["portable_data_hash"] for i in items}
+ need_migrate = [img for img in old_images
+ if uuid_to_pdh[img["collection"]] not in already_migrated]
+
+ logger.info("Already migrated %i images", len(already_migrated))
+ logger.info("Need to migrate %i images", len(need_migrate))
+
+ success = []
+ failures = []
+ for old_image in need_migrate:
+ if uuid_to_pdh[old_image["collection"]] in already_migrated:
+ continue
+
+ logger.info("Migrating %s:%s (%s)", old_image["repo"], old_image["tag"], old_image["collection"])
+
+ oldcol = CollectionReader(old_image["collection"])
+ tarfile = oldcol.keys()[0]
+
+ try:
+ varlibdocker = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile() as envfile:
+ envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+ if "ARVADOS_API_HOST_INSECURE" in os.environ:
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.flush()
+
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env-file", envfile.name,
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/migrate-docker19",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ oldcol.api_response()["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
+
+ migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+ if migrated:
+ newcol = CollectionReader(migrated.group(1))
+
+ api_client.links().create(body={"link": {
+ 'owner_uuid': sys_uuid,
+ 'link_class': arvados.commands.keepdocker._migration_link_class,
+ 'name': arvados.commands.keepdocker._migration_link_name,
+ 'tail_uuid': oldcol.portable_data_hash(),
+ 'head_uuid': newcol.portable_data_hash()
+ }}).execute(num_retries=3)
+
+ logger.info("Migrated '%s' to '%s'", oldcol.portable_data_hash(), newcol.portable_data_hash())
+ already_migrated.add(oldcol.portable_data_hash())
+ success.append(old_image["collection"])
+ else:
+ logger.error("Error migrating '%s'", old_image["collection"])
+ failures.append(old_image["collection"])
+ except Exception as e:
+ logger.exception("Migration failed")
+ failures.append(old_image["collection"])
+ finally:
+ shutil.rmtree(varlibdocker)
+
+ logger.info("Successfully migrated %i images", len(success))
+ if failures:
+ logger.error("Failure migrating images: %s", failures)
+
+
+def main(arguments=None, stdout=sys.stdout):