From ac69bffe0f16b65d06efca4b7544c143dee0ab6e Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Fri, 12 Jul 2024 11:17:13 -0400 Subject: [PATCH] 21935/15397: Remove arv-migrate-docker19 This tool was written to aid a migration that became necessary with the release of Docker 1.10 in February 2016. Its time has passed. Arvados-DCO-1.1-Signed-off-by: Brett Smith --- cmd/arvados-package/fpm.go | 1 - doc/_config.yml | 1 - doc/admin/upgrading.html.textile.liquid | 4 + .../migrate-docker19.html.textile.liquid | 39 --- docker/migrate-docker19/Dockerfile | 35 --- docker/migrate-docker19/build.sh | 6 - docker/migrate-docker19/dnd.sh | 102 ------ docker/migrate-docker19/migrate.sh | 113 ------- lib/install/deps.go | 1 - sdk/python/arvados/commands/migrate19.py | 292 ------------------ sdk/python/bin/arv-migrate-docker19 | 7 - sdk/python/setup.py | 1 - 12 files changed, 4 insertions(+), 598 deletions(-) delete mode 100644 doc/install/migrate-docker19.html.textile.liquid delete mode 100644 docker/migrate-docker19/Dockerfile delete mode 100755 docker/migrate-docker19/build.sh delete mode 100755 docker/migrate-docker19/dnd.sh delete mode 100755 docker/migrate-docker19/migrate.sh delete mode 100644 sdk/python/arvados/commands/migrate19.py delete mode 100755 sdk/python/bin/arv-migrate-docker19 diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go index 0b13b15c1e..dab215561b 100644 --- a/cmd/arvados-package/fpm.go +++ b/cmd/arvados-package/fpm.go @@ -117,7 +117,6 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ "/usr/bin/arv-get", "/usr/bin/arv-keepdocker", "/usr/bin/arv-ls", - "/usr/bin/arv-migrate-docker19", "/usr/bin/arv-normalize", "/usr/bin/arv-put", "/usr/bin/arv-ws", diff --git a/doc/_config.yml b/doc/_config.yml index c8cf7b0dd9..83627bb82d 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -191,7 +191,6 @@ navbar: - admin/spot-instances.html.textile.liquid - admin/cloudtest.html.textile.liquid - Other: - - install/migrate-docker19.html.textile.liquid - admin/upgrade-crunch2.html.textile.liquid installguide: - Overview: diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid index 0d83c6a187..c2efa17e5e 100644 --- a/doc/admin/upgrading.html.textile.liquid +++ b/doc/admin/upgrading.html.textile.liquid @@ -109,6 +109,10 @@ h3. WebDAV service uses @/var/cache@ for file content If the previously supported @MaxBlockEntries@ config is present, remove it to avoid warning messages at startup. +h3. @arv-migrate-docker19@ tool removed + +The @arv-migrate-docker19@ tool that updates images from Docker 1.9 to be used with Docker 1.10+ (released February 2016) has been removed. In the unlikely event you still need to "run this migration":https://doc.arvados.org/v2.7/install/migrate-docker19.html, please do so before you upgrade to Arvados 3.0. + h2(#v2_7_4). v2.7.4 (2024-07-08) "previous: Upgrading to 2.7.3":#v2_7_3 diff --git a/doc/install/migrate-docker19.html.textile.liquid b/doc/install/migrate-docker19.html.textile.liquid deleted file mode 100644 index 7b7e2a83cf..0000000000 --- a/doc/install/migrate-docker19.html.textile.liquid +++ /dev/null @@ -1,39 +0,0 @@ ---- -layout: default -navsection: admin -title: Migrating from Docker 1.9 -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -If you have an existing Arvados installation using Docker 1.9 and wish to update to Docker 1.10+, you must migrate the Docker images stored in Keep. - -The @arv-migrate-docker19@ tool converts Docker images stored in Arvados from image format v1 (Docker <= 1.9) to image format v2 (Docker >= 1.10). - -Requires Docker running on the local host (can be either 1.9 or 1.10+). Linux kernel >= 3.18-rc6 to support overlayfs. - -Usage: - -# Install arvados/migrate-docker19 image: @docker pull arvados/migrate-docker19:1.0@. If you're unable to do this, you can run @arvados/docker/migrate-docker19/build.sh@ to create @arvados/migrate-docker19@ Docker image. -# Make sure you have the right modules installed: @sudo modprobe overlayfs bridge br_netfilter nf_nat@ -# Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate. -# Your temporary directory should have the size of all layers of the biggest image in the cluster, this is hard to estimate, but you can start with five times that size. You can set up a different directory by using the @--tempdir@ switch. Make sure that the user running the docker daemon has permissions to write in that directory. -# Run @arv-migrate-docker19 --dry-run@ from the Arvados Python SDK on the host (not in a container). This will print out some information useful for the migration. -# Finally to make the migration run @arv-migrate-docker19@ from the Arvados Python SDK on the host (not in a container). - -This will query Arvados for v1 format Docker images. For each image that does not already have a corresponding v2 format image (as indicated by a docker_image_migration tag) it will perform the following process: - -i) download the image from Arvados -ii) load it into Docker -iii) update the Docker version, which updates the image -iv) save the v2 format image and upload to Arvados -v) create a migration link - -Once the Docker images in Keep have been migrated, upgrade the version of Docker used across the cluster. Finally, update the API server configuration from "v1" to "v2" to reflect the supported Docker image version: - -
-docker_image_formats: ["v2"]
-
diff --git a/docker/migrate-docker19/Dockerfile b/docker/migrate-docker19/Dockerfile deleted file mode 100644 index 23bb63547b..0000000000 --- a/docker/migrate-docker19/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -FROM debian:8 - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 && \ - gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \ - apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \ - apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - -VOLUME /var/lib/docker - -RUN mkdir -p /etc/apt/sources.list.d && \ - echo deb http://apt.arvados.org/jessie jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \ - apt-get clean && \ - apt-get update && \ - apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \ - git curl python-arvados-python-client apt-transport-https ca-certificates && \ - apt-get clean - -RUN echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \ - apt-get update && \ - apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \ - docker-engine=1.9.1-0~jessie && \ - apt-get clean - -RUN mkdir /root/pkgs && \ - cd /root/pkgs && \ - curl -L -O https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.13.1-0~debian-jessie_amd64.deb && \ - curl -L -O http://httpredir.debian.org/debian/pool/main/libt/libtool/libltdl7_2.4.2-1.11+b1_amd64.deb - -ADD migrate.sh dnd.sh /root/ diff --git a/docker/migrate-docker19/build.sh b/docker/migrate-docker19/build.sh deleted file mode 100755 index 5d76ec7faa..0000000000 --- a/docker/migrate-docker19/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -exec docker build -t arvados/migrate-docker19:1.0 . diff --git a/docker/migrate-docker19/dnd.sh b/docker/migrate-docker19/dnd.sh deleted file mode 100755 index 703a124280..0000000000 --- a/docker/migrate-docker19/dnd.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -# Taken from https://github.com/jpetazzo/dind - -exec 2>&1 - -# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver -dmsetup mknodes - -: {LOG:=stdio} - -# First, make sure that cgroups are mounted correctly. -CGROUP=/sys/fs/cgroup -[ -d $CGROUP ] || mkdir $CGROUP - -if mountpoint -q $CGROUP ; then - true -else - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP -fi - -if ! mountpoint -q $CGROUP ; then - echo "Could not find or mount cgroups. Tried /sys/fs/cgroup and /cgroup. Did you use --privileged?" - exit 1 -fi - -if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security -then - mount -t securityfs none /sys/kernel/security || { - echo "Could not mount /sys/kernel/security." - echo "AppArmor detection and --privileged mode might break." - } -fi - -# Mount the cgroup hierarchies exactly as they are in the parent system. -for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) -do - [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS - mountpoint -q $CGROUP/$SUBSYS || - mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS - - # The two following sections address a bug which manifests itself - # by a cryptic "lxc-start: no ns_cgroup option specified" when - # trying to start containers withina container. - # The bug seems to appear when the cgroup hierarchies are not - # mounted on the exact same directories in the host, and in the - # container. - - # Named, control-less cgroups are mounted with "-o name=foo" - # (and appear as such under /proc//cgroup) but are usually - # mounted on a directory named "foo" (without the "name=" prefix). - # Systemd and OpenRC (and possibly others) both create such a - # cgroup. To avoid the aforementioned bug, we symlink "foo" to - # "name=foo". This shouldn't have any adverse effect. - #echo $SUBSYS | grep -q ^name= && { - # NAME=$(echo $SUBSYS | sed s/^name=//) - # ln -s $SUBSYS $CGROUP/$NAME - #} - - # Likewise, on at least one system, it has been reported that - # systemd would mount the CPU and CPU accounting controllers - # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" - # but on a directory called "cpu,cpuacct" (note the inversion - # in the order of the groups). This tries to work around it. - [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct -done - -# Note: as I write those lines, the LXC userland tools cannot setup -# a "sub-container" properly if the "devices" cgroup is not in its -# own hierarchy. Let's detect this and issue a warning. -grep -q :devices: /proc/1/cgroup || - echo "WARNING: the 'devices' cgroup should be in its own hierarchy." -grep -qw devices /proc/1/cgroup || - echo "WARNING: it looks like the 'devices' cgroup is not mounted." - -# Now, close extraneous file descriptors. -pushd /proc/self/fd >/dev/null -for FD in * -do - case "$FD" in - # Keep stdin/stdout/stderr - [012]) - ;; - # Nuke everything else - *) - eval exec "$FD>&-" - ;; - esac -done -popd >/dev/null - - -# If a pidfile is still around (for example after a container restart), -# delete it so that docker can start. -rm -rf /var/run/docker.pid - -read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat - -exec docker daemon --storage-driver=$1 $DOCKER_DAEMON_ARGS diff --git a/docker/migrate-docker19/migrate.sh b/docker/migrate-docker19/migrate.sh deleted file mode 100755 index 76fe823394..0000000000 --- a/docker/migrate-docker19/migrate.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -# This script is called by arv-migrate-docker19 to perform the actual migration -# of a single image. This works by running Docker-in-Docker (dnd.sh) to -# download the image using Docker 1.9 and then upgrading to Docker 1.13 and -# uploading the converted image. - -# When using bash in pid 1 and using "trap on EXIT" -# it will sometimes go into an 100% CPU infinite loop. -# -# Using workaround from here: -# -# https://github.com/docker/docker/issues/4854 -if [ "$$" = 1 ]; then - $0 "$@" - exit $? -fi - -# -x show script -# -e exit on error -# -o pipefail use exit code from 1st failure in pipeline, not last -set -x -e -o pipefail - -image_tar_keepref=$1 -image_id=$2 -image_repo=$3 -image_tag=$4 -project_uuid=$5 -graph_driver=$6 - -if [[ "$image_repo" = "" ]] ; then - image_repo=none - image_tag=latest -fi - -# Print free space in /var/lib/docker -function freespace() { - df -B1 /var/lib/docker | tail -n1 | sed 's/ */ /g' | cut -d' ' -f4 -} - -# Run docker-in-docker script and then wait for it to come up -function start_docker { - /root/dnd.sh $graph_driver & - for i in $(seq 1 10) ; do - if docker version >/dev/null 2>/dev/null ; then - return - fi - sleep 1 - done - false -} - -# Kill docker from pid then wait for it to be down -function kill_docker { - if test -f /var/run/docker.pid ; then - kill $(cat /var/run/docker.pid) - fi - for i in $(seq 1 10) ; do - if ! docker version >/dev/null 2>/dev/null ; then - return - fi - sleep 1 - done - false -} - -# Ensure that we clean up docker graph and/or lingering cache files on exit -function cleanup { - kill_docker - rm -rf /var/lib/docker/* - rm -rf /root/.cache/arvados/docker/* - echo "Available space after cleanup is $(freespace)" -} - -trap cleanup EXIT - -start_docker - -echo "Initial available space is $(freespace)" - -arv-get $image_tar_keepref | docker load - - -docker tag $image_id $image_repo:$image_tag - -docker images -a - -kill_docker - -echo "Available space after image load is $(freespace)" - -cd /root/pkgs -dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb - -echo "Available space after image upgrade is $(freespace)" - -start_docker - -docker images -a - -if [[ "$image_repo" = "none" ]] ; then - image_repo=$(docker images -a --no-trunc | sed 's/ */ /g' | grep ^none | cut -d' ' -f3) - image_tag="" -fi - -UUID=$(arv-keepdocker --force-image-format --project-uuid=$project_uuid $image_repo $image_tag) - -echo "Available space after arv-keepdocker is $(freespace)" - -echo "Migrated uuid is $UUID" diff --git a/lib/install/deps.go b/lib/install/deps.go index 5909499bac..a64219b47f 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -774,7 +774,6 @@ rsync -a --delete-after "$tmp/build/" "$dst/" {"lib/python/bin/arv-get"}, {"lib/python/bin/arv-keepdocker"}, {"lib/python/bin/arv-ls"}, - {"lib/python/bin/arv-migrate-docker19"}, {"lib/python/bin/arv-normalize"}, {"lib/python/bin/arv-put"}, {"lib/python/bin/arv-ws"}, diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py deleted file mode 100644 index 7bcc784fd7..0000000000 --- a/sdk/python/arvados/commands/migrate19.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import time -import sys -import logging -import shutil -import tempfile -import os -import subprocess -import re - -import arvados -import arvados.commands.keepdocker -from arvados._version import __version__ -from arvados.collection import CollectionReader -from .. import util - -logger = logging.getLogger('arvados.migrate-docker19') -logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG') - else logging.INFO) - -_migration_link_class = 'docker_image_migration' -_migration_link_name = 'migrate_1.9_1.10' - -class MigrationFailed(Exception): - pass - -@util._deprecated('3.0') -def main(arguments=None): - """Docker image format migration tool for Arvados. - - This converts Docker images stored in Arvados from image format v1 - (Docker <= 1.9) to image format v2 (Docker >= 1.10). - - Requires Docker running on the local host. - - Usage: - - 1) Run arvados/docker/migrate-docker19/build.sh to create - arvados/migrate-docker19 Docker image. - - 2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate. - - 3) Run arv-migrate-docker19 from the Arvados Python SDK on the host (not in a container). - - This will query Arvados for v1 format Docker images. For each image that - does not already have a corresponding v2 format image (as indicated by a - docker_image_migration tag) it will perform the following process: - - i) download the image from Arvados - ii) load it into Docker - iii) update the Docker version, which updates the image - iv) save the v2 format image and upload to Arvados - v) create a migration link - - """ - - migrate19_parser = argparse.ArgumentParser() - migrate19_parser.add_argument( - '--version', action='version', version="%s %s" % (sys.argv[0], __version__), - help='Print version and exit.') - migrate19_parser.add_argument( - '--verbose', action="store_true", help="Print stdout/stderr even on success") - migrate19_parser.add_argument( - '--force', action="store_true", help="Try to migrate even if there isn't enough space") - - migrate19_parser.add_argument( - '--storage-driver', type=str, default="overlay", - help="Docker storage driver, e.g. aufs, overlay, vfs") - - exgroup = migrate19_parser.add_mutually_exclusive_group() - exgroup.add_argument( - '--dry-run', action='store_true', help="Print number of pending migrations.") - exgroup.add_argument( - '--print-unmigrated', action='store_true', - default=False, help="Print list of images needing migration.") - - migrate19_parser.add_argument('--tempdir', help="Set temporary directory") - - migrate19_parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), - default=None, help="List of images to be migrated") - - args = migrate19_parser.parse_args(arguments) - - if args.tempdir: - tempfile.tempdir = args.tempdir - - if args.verbose: - logger.setLevel(logging.DEBUG) - - only_migrate = None - if args.infile: - only_migrate = set() - for l in args.infile: - only_migrate.add(l.strip()) - - api_client = arvados.api() - - user = api_client.users().current().execute() - if not user['is_admin']: - raise Exception("This command requires an admin token") - sys_uuid = user['uuid'][:12] + '000000000000000' - - images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3) - - is_new = lambda img: img['dockerhash'].startswith('sha256:') - - count_new = 0 - old_images = [] - for uuid, img in images: - if img["dockerhash"].startswith("sha256:"): - continue - key = (img["repo"], img["tag"], img["timestamp"]) - old_images.append(img) - - migration_links = arvados.util.keyset_list_all( - api_client.links().list, - filters=[ - ['link_class', '=', _migration_link_class], - ['name', '=', _migration_link_name], - ], - order='uuid') - - already_migrated = set() - for m in migration_links: - already_migrated.add(m["tail_uuid"]) - - items = arvados.util.keyset_list_all( - api_client.collections().list, - filters=[["uuid", "in", [img["collection"] for img in old_images]]], - select=["uuid", "portable_data_hash", "manifest_text", "owner_uuid"], - order='uuid') - uuid_to_collection = {i["uuid"]: i for i in items} - - need_migrate = {} - totalbytes = 0 - biggest = 0 - biggest_pdh = None - for img in old_images: - i = uuid_to_collection[img["collection"]] - pdh = i["portable_data_hash"] - if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate): - need_migrate[pdh] = img - with CollectionReader(i["manifest_text"]) as c: - size = list(c.values())[0].size() - if size > biggest: - biggest = size - biggest_pdh = pdh - totalbytes += size - - - if args.storage_driver == "vfs": - will_need = (biggest*20) - else: - will_need = (biggest*2.5) - - if args.print_unmigrated: - only_migrate = set() - for pdh in need_migrate: - print(pdh) - return - - logger.info("Already migrated %i images", len(already_migrated)) - logger.info("Need to migrate %i images", len(need_migrate)) - logger.info("Using tempdir %s", tempfile.gettempdir()) - logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest>>20) - logger.info("Total data to migrate about %i MiB", totalbytes>>20) - - df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()]) - ln = df_out.splitlines()[1] - filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1) - if int(available) <= will_need: - logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)>>20, int(will_need)>>20) - if not args.force: - exit(1) - else: - logger.warn("--force provided, will migrate anyway") - - if args.dry_run: - return - - success = [] - failures = [] - count = 1 - for old_image in list(need_migrate.values()): - if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated: - continue - - oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"]) - tarfile = list(oldcol.keys())[0] - - logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"], - old_image["tag"], old_image["collection"], list(oldcol.values())[0].size()>>20) - count += 1 - start = time.time() - - varlibdocker = tempfile.mkdtemp() - dockercache = tempfile.mkdtemp() - try: - with tempfile.NamedTemporaryFile() as envfile: - envfile.write("ARVADOS_API_HOST=%s\n" % (arvados.config.get("ARVADOS_API_HOST"))) - envfile.write("ARVADOS_API_TOKEN=%s\n" % (arvados.config.get("ARVADOS_API_TOKEN"))) - if arvados.config.get("ARVADOS_API_HOST_INSECURE"): - envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (arvados.config.get("ARVADOS_API_HOST_INSECURE"))) - envfile.flush() - - dockercmd = ["docker", "run", - "--privileged", - "--rm", - "--env-file", envfile.name, - "--volume", "%s:/var/lib/docker" % varlibdocker, - "--volume", "%s:/root/.cache/arvados/docker" % dockercache, - "arvados/migrate-docker19:1.0", - "/root/migrate.sh", - "%s/%s" % (old_image["collection"], tarfile), - tarfile[0:40], - old_image["repo"], - old_image["tag"], - uuid_to_collection[old_image["collection"]]["owner_uuid"], - args.storage_driver] - - proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - - initial_space = re.search(r"Initial available space is (\d+)", out) - imgload_space = re.search(r"Available space after image load is (\d+)", out) - imgupgrade_space = re.search(r"Available space after image upgrade is (\d+)", out) - keepdocker_space = re.search(r"Available space after arv-keepdocker is (\d+)", out) - cleanup_space = re.search(r"Available space after cleanup is (\d+)", out) - - if initial_space: - isp = int(initial_space.group(1)) - logger.info("Available space initially: %i MiB", (isp)/(2**20)) - if imgload_space: - sp = int(imgload_space.group(1)) - logger.debug("Used after load: %i MiB", (isp-sp)/(2**20)) - if imgupgrade_space: - sp = int(imgupgrade_space.group(1)) - logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20)) - if keepdocker_space: - sp = int(keepdocker_space.group(1)) - logger.info("Used after upload: %i MiB", (isp-sp)/(2**20)) - - if cleanup_space: - sp = int(cleanup_space.group(1)) - logger.debug("Available after cleanup: %i MiB", (sp)/(2**20)) - - if proc.returncode != 0: - logger.error("Failed with return code %i", proc.returncode) - logger.error("--- Stdout ---\n%s", out) - logger.error("--- Stderr ---\n%s", err) - raise MigrationFailed() - - if args.verbose: - logger.info("--- Stdout ---\n%s", out) - logger.info("--- Stderr ---\n%s", err) - - migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out) - if migrated: - newcol = CollectionReader(migrated.group(1)) - - api_client.links().create(body={"link": { - 'owner_uuid': sys_uuid, - 'link_class': _migration_link_class, - 'name': _migration_link_name, - 'tail_uuid': oldcol.portable_data_hash(), - 'head_uuid': newcol.portable_data_hash() - }}).execute(num_retries=3) - - logger.info("Migrated '%s' (%s) to '%s' (%s) in %is", - oldcol.portable_data_hash(), old_image["collection"], - newcol.portable_data_hash(), migrated.group(1), - time.time() - start) - already_migrated.add(oldcol.portable_data_hash()) - success.append(old_image["collection"]) - else: - logger.error("Error migrating '%s'", old_image["collection"]) - failures.append(old_image["collection"]) - except Exception as e: - logger.error("Failed to migrate %s in %is", old_image["collection"], time.time() - start, - exc_info=(not isinstance(e, MigrationFailed))) - failures.append(old_image["collection"]) - finally: - shutil.rmtree(varlibdocker) - shutil.rmtree(dockercache) - - logger.info("Successfully migrated %i images", len(success)) - if failures: - logger.error("Failed to migrate %i images", len(failures)) diff --git a/sdk/python/bin/arv-migrate-docker19 b/sdk/python/bin/arv-migrate-docker19 deleted file mode 100755 index 6aee15254a..0000000000 --- a/sdk/python/bin/arv-migrate-docker19 +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -from arvados.commands.migrate19 import main -main() diff --git a/sdk/python/setup.py b/sdk/python/setup.py index dc10269060..9e1c1839f8 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -97,7 +97,6 @@ setup(name='arvados-python-client', 'bin/arv-get', 'bin/arv-keepdocker', 'bin/arv-ls', - 'bin/arv-migrate-docker19', 'bin/arv-federation-migrate', 'bin/arv-normalize', 'bin/arv-put', -- 2.39.5