From 6058f336a4562b6d6780c4fcfc4d28b6b2a51948 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 6 Mar 2017 14:52:55 -0500 Subject: [PATCH] 8567: Move out of tools/ into sdk/python and docker/docker19-migrate. --- .../docker19-migrate}/Dockerfile | 0 .../docker19-migrate}/README | 2 +- docker/docker19-migrate/build.sh | 2 + .../docker => docker/docker19-migrate}/dnd.sh | 0 .../docker19-migrate}/migrate.sh | 0 sdk/python/arvados/commands/keepdocker.py | 133 +++++++++++------- tools/docker-migrator/build.sh | 2 - tools/docker-migrator/migrate.py | 89 ------------ 8 files changed, 85 insertions(+), 143 deletions(-) rename {tools/docker-migrator/docker => docker/docker19-migrate}/Dockerfile (100%) rename {tools/docker-migrator => docker/docker19-migrate}/README (96%) create mode 100755 docker/docker19-migrate/build.sh rename {tools/docker-migrator/docker => docker/docker19-migrate}/dnd.sh (100%) rename {tools/docker-migrator/docker => docker/docker19-migrate}/migrate.sh (100%) delete mode 100755 tools/docker-migrator/build.sh delete mode 100755 tools/docker-migrator/migrate.py diff --git a/tools/docker-migrator/docker/Dockerfile b/docker/docker19-migrate/Dockerfile similarity index 100% rename from tools/docker-migrator/docker/Dockerfile rename to docker/docker19-migrate/Dockerfile diff --git a/tools/docker-migrator/README b/docker/docker19-migrate/README similarity index 96% rename from tools/docker-migrator/README rename to docker/docker19-migrate/README index d94eb88632..59fa6260ad 100644 --- a/tools/docker-migrator/README +++ b/docker/docker19-migrate/README @@ -11,7 +11,7 @@ Usage: 2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate. -3) Run ./migrate.py +3) Run arv-migrate-docker19 This will query Arvados for v1 format Docker images. For each image that does not already have a corresponding v2 format image (as indicated by a diff --git a/docker/docker19-migrate/build.sh b/docker/docker19-migrate/build.sh new file mode 100755 index 0000000000..e6dc998c62 --- /dev/null +++ b/docker/docker19-migrate/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec docker build -t arvados/docker19-migrate . diff --git a/tools/docker-migrator/docker/dnd.sh b/docker/docker19-migrate/dnd.sh similarity index 100% rename from tools/docker-migrator/docker/dnd.sh rename to docker/docker19-migrate/dnd.sh diff --git a/tools/docker-migrator/docker/migrate.sh b/docker/docker19-migrate/migrate.sh similarity index 100% rename from tools/docker-migrator/docker/migrate.sh rename to docker/docker19-migrate/migrate.sh diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py index 3ffb7f36b6..7e1b170674 100644 --- a/sdk/python/arvados/commands/keepdocker.py +++ b/sdk/python/arvados/commands/keepdocker.py @@ -11,6 +11,7 @@ import subprocess import sys import tarfile import tempfile +import shutil import _strptime from operator import itemgetter @@ -20,10 +21,17 @@ import arvados import arvados.util import arvados.commands._util as arv_cmd import arvados.commands.put as arv_put +from arvados.collection import CollectionReader import ciso8601 +import logging +import arvados.config from arvados._version import __version__ +logger = logging.getLogger('arvados.keepdocker') +logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG') + else logging.INFO) + EARLIEST_DATETIME = datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0) STAT_CACHE_ERRORS = (IOError, OSError, ValueError) @@ -103,15 +111,15 @@ def docker_image_format(image_hash): def docker_image_compatible(api, image_hash): supported = api._rootDesc.get('dockerImageFormats', []) if not supported: - print >>sys.stderr, "arv-keepdocker: warning: server does not specify supported image formats (see docker_image_formats in server config). Continuing." + logger.warn("server does not specify supported image formats (see docker_image_formats in server config). Continuing.") return True fmt = docker_image_format(image_hash) if fmt in supported: return True else: - print >>sys.stderr, "arv-keepdocker: image format is {!r} " \ - "but server supports only {!r}".format(fmt, supported) + logger.error("image format is {!r} " \ + "but server supports only {!r}".format(fmt, supported)) return False def docker_images(): @@ -332,59 +340,82 @@ def _uuid2pdh(api, uuid): _migration_link_class = 'docker_image_migration' _migration_link_name = 'migrate_1.9_1.10' -def _migrate19_link(api, root_uuid, old_uuid, new_uuid): - old_pdh = _uuid2pdh(api, old_uuid) - new_pdh = _uuid2pdh(api, new_uuid) - if not api.links().list(filters=[ - ['owner_uuid', '=', root_uuid], - ['link_class', '=', _migration_link_class], - ['name', '=', _migration_link_name], - ['tail_uuid', '=', old_pdh], - ['head_uuid', '=', new_pdh]]).execute()['items']: - print >>sys.stderr, 'Creating migration link {} -> {}: '.format( - old_pdh, new_pdh), - link = api.links().create(body={ - 'owner_uuid': root_uuid, - 'link_class': _migration_link_class, - 'name': _migration_link_name, - 'tail_uuid': old_pdh, - 'head_uuid': new_pdh, - }).execute() - print >>sys.stderr, '{}'.format(link['uuid']) - return link def migrate19(): - api = arvados.api('v1') - user = api.users().current().execute() - if not user['is_admin']: - raise Exception("This command requires an admin token") - root_uuid = user['uuid'][:12] + '000000000000000' - new_image_uuids = {} - images = list_images_in_arv(api, 2) + api_client = arvados.api() + + images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3) + is_new = lambda img: img['dockerhash'].startswith('sha256:') count_new = 0 + old_images = [] for uuid, img in images: - if not re.match(r'^[0-9a-f]{64}$', img["tag"]): + if img["dockerhash"].startswith("sha256:"): continue - key = (img["repo"], img["tag"]) - if is_new(img) and key not in new_image_uuids: - count_new += 1 - new_image_uuids[key] = uuid + key = (img["repo"], img["tag"], img["timestamp"]) + old_images.append(img) - count_migrations = 0 - new_links = [] - for uuid, img in images: - key = (img['repo'], img['tag']) - if not is_new(img) and key in new_image_uuids: - count_migrations += 1 - link = _migrate19_link(api, root_uuid, uuid, new_image_uuids[key]) - if link: - new_links.append(link) + migration_links = arvados.util.list_all(api_client.links().list, filters=[ + ['link_class', '=', _migration_link_class], + ['name', '=', _migration_link_name], + ]) + + already_migrated = set() + for m in migration_links: + already_migrated.add(m["tail_uuid"]) + + need_migrate = [img for img in old_images if img["collection"] not in already_migrated] + + logger.info("Already migrated %i images", len(already_migrated)) + logger.info("Need to migrate %i images", len(need_migrate)) + + for old_image in need_migrate: + logger.info("Migrating %s", old_image["collection"]) + + col = CollectionReader(old_image["collection"]) + tarfile = col.keys()[0] + + try: + varlibdocker = tempfile.mkdtemp() + with tempfile.NamedTemporaryFile() as envfile: + envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"])) + envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"])) + envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"])) + envfile.flush() + + dockercmd = ["docker", "run", + "--privileged", + "--rm", + "--env-file", envfile.name, + "--volume", "%s:/var/lib/docker" % varlibdocker, + "arvados/docker19-migrate", + "/root/migrate.sh", + "%s/%s" % (old_image["collection"], tarfile), + tarfile[0:40], + old_image["repo"], + old_image["tag"], + col.api_response()["owner_uuid"]] + + out = subprocess.check_output(dockercmd) + + new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out) + api_client.links().create(body={"link": { + 'owner_uuid': col.api_response()["owner_uuid"], + 'link_class': arvados.commands.keepdocker._migration_link_class, + 'name': arvados.commands.keepdocker._migration_link_name, + 'tail_uuid': old_image["collection"], + 'head_uuid': new_collection.group(1) + }}).execute(num_retries=3) + + logger.info("Migrated '%s' to '%s'", old_image["collection"], new_collection.group(1)) + except Exception as e: + logger.exception("Migration failed") + finally: + shutil.rmtree(varlibdocker) + + logger.info("All done") - print >>sys.stderr, "=== {} new-format images, {} migrations detected, " \ - "{} links added.".format(count_new, count_migrations, len(new_links)) - return new_links def main(arguments=None, stdout=sys.stdout): args = arg_parser.parse_args(arguments) @@ -405,15 +436,15 @@ def main(arguments=None, stdout=sys.stdout): try: image_hash = find_one_image_hash(args.image, args.tag) except DockerError as error: - print >>sys.stderr, "arv-keepdocker:", error.message + logger.error(error.message) sys.exit(1) if not docker_image_compatible(api, image_hash): if args.force_image_format: - print >>sys.stderr, "arv-keepdocker: forcing incompatible image" + logger.warn("forcing incompatible image") else: - print >>sys.stderr, "arv-keepdocker: refusing to store " \ - "incompatible format (use --force-image-format to override)" + logger.error("refusing to store " \ + "incompatible format (use --force-image-format to override)") sys.exit(1) image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh deleted file mode 100755 index ecef09dedd..0000000000 --- a/tools/docker-migrator/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec docker build -t arvados/docker19-migrate docker diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py deleted file mode 100755 index ec10b75734..0000000000 --- a/tools/docker-migrator/migrate.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -import arvados -import arvados.util -from arvados.collection import CollectionReader -import arvados.commands.keepdocker -import re -import subprocess -import os -import tempfile -import shutil - -from pprint import pprint - -def main(): - api_client = arvados.api() - - images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3) - - is_new = lambda img: img['dockerhash'].startswith('sha256:') - - count_new = 0 - old_images = [] - for uuid, img in images: - if img["dockerhash"].startswith("sha256:"): - continue - key = (img["repo"], img["tag"], img["timestamp"]) - old_images.append(img) - - migration_links = arvados.util.list_all(api_client.links().list, filters=[ - ['link_class', '=', arvados.commands.keepdocker._migration_link_class], - ['name', '=', arvados.commands.keepdocker._migration_link_name], - ]) - - already_migrated = set() - for m in migration_links: - already_migrated.add(m["tail_uuid"]) - - need_migrate = [img for img in old_images if img["collection"] not in already_migrated] - - print "Already migrated %i images" % (len(already_migrated)) - print "Need to migrate %i images" % (len(need_migrate)) - - for old_image in need_migrate: - print "Migrating %s" % (old_image["collection"]) - - col = CollectionReader(old_image["collection"]) - tarfile = col.keys()[0] - - try: - varlibdocker = tempfile.mkdtemp() - with tempfile.NamedTemporaryFile() as envfile: - envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"])) - envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"])) - envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"])) - envfile.flush() - - dockercmd = ["docker", "run", - "--privileged", - "--rm", - "--env-file", envfile.name, - "--volume", "%s:/var/lib/docker" % varlibdocker, - "arvados/docker19-migrate", - "/root/migrate.sh", - "%s/%s" % (old_image["collection"], tarfile), - tarfile[0:40], - old_image["repo"], - old_image["tag"], - col.api_response()["owner_uuid"]] - - out = subprocess.check_output(dockercmd) - - new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out) - api_client.links().create(body={"link": { - 'owner_uuid': col.api_response()["owner_uuid"], - 'link_class': arvados.commands.keepdocker._migration_link_class, - 'name': arvados.commands.keepdocker._migration_link_name, - 'tail_uuid': old_image["collection"], - 'head_uuid': new_collection.group(1) - }}).execute(num_retries=3) - - print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1)) - finally: - shutil.rmtree(varlibdocker) - - print "All done" - - -main() -- 2.30.2