From 433c48edb1523809bb8d43e4fc40ebe1a177e103 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 30 Mar 2017 10:07:52 -0400 Subject: [PATCH] 8567: Improve disk space accounting. Default to use overlay storage driver (but user can override). --- docker/migrate-docker19/build.sh | 2 +- docker/migrate-docker19/dnd.sh | 2 +- docker/migrate-docker19/migrate.sh | 27 +++++++--- sdk/python/arvados/commands/migrate19.py | 65 ++++++++++++++++++++++-- 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/docker/migrate-docker19/build.sh b/docker/migrate-docker19/build.sh index 3a36cd495a..e563bfc7e0 100755 --- a/docker/migrate-docker19/build.sh +++ b/docker/migrate-docker19/build.sh @@ -1,2 +1,2 @@ #!/bin/sh -exec docker build -t arvados/migrate-docker19 . +exec docker build -t arvados/migrate-docker19:1.0 . diff --git a/docker/migrate-docker19/dnd.sh b/docker/migrate-docker19/dnd.sh index ec6f1e3e12..f253f0be66 100755 --- a/docker/migrate-docker19/dnd.sh +++ b/docker/migrate-docker19/dnd.sh @@ -96,4 +96,4 @@ rm -rf /var/run/docker.pid read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat -exec docker daemon --storage-driver=vfs $DOCKER_DAEMON_ARGS +exec docker daemon --storage-driver=$1 $DOCKER_DAEMON_ARGS diff --git a/docker/migrate-docker19/migrate.sh b/docker/migrate-docker19/migrate.sh index 8e39be4567..350ae4ce56 100755 --- a/docker/migrate-docker19/migrate.sh +++ b/docker/migrate-docker19/migrate.sh @@ -2,8 +2,19 @@ set -e +image_tar_keepref=$1 +image_id=$2 +image_repo=$3 +image_tag=$4 +project_uuid=$5 +graph_driver=$6 + +function freespace() { + df -B1 /var/lib/docker | tail -n1 | sed 's/ */ /g' | cut -d' ' -f4 +} + function start_docker { - /root/dnd.sh & + /root/dnd.sh $graph_driver & for i in $(seq 1 10) ; do if docker version >/dev/null 2>/dev/null ; then return @@ -27,20 +38,18 @@ function kill_docker { } function cleanup { + trap EXIT kill_docker rm -rf /var/lib/docker/* rm -rf /root/.cache/arvados/docker/* + echo "Available space after cleanup is $(freespace)" } trap cleanup EXIT start_docker -image_tar_keepref=$1 -image_id=$2 -image_repo=$3 -image_tag=$4 -project_uuid=$5 +echo "Initial available space is $(freespace)" arv-get $image_tar_keepref | docker load @@ -50,13 +59,19 @@ docker images -a kill_docker +echo "Available space after image load is $(freespace)" + cd /root/pkgs dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb +echo "Available space after image upgrade is $(freespace)" + start_docker docker images -a UUID=$(arv-keepdocker --force-image-format --project-uuid=$project_uuid $image_repo $image_tag) +echo "Available space after arv-keepdocker is $(freespace)" + echo "Migrated uuid is $UUID" diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py index 4be99961ba..a9768a0374 100644 --- a/sdk/python/arvados/commands/migrate19.py +++ b/sdk/python/arvados/commands/migrate19.py @@ -56,6 +56,14 @@ def main(arguments=None): migrate19_parser.add_argument( '--version', action='version', version="%s %s" % (sys.argv[0], __version__), help='Print version and exit.') + migrate19_parser.add_argument( + '--verbose', action="store_true", help="Print stdout/stderr even on success") + migrate19_parser.add_argument( + '--force', action="store_true", help="Try to migrate even if there isn't enough space") + + migrate19_parser.add_argument( + '--storage-driver', type=str, default="overlay", + help="Docker storage driver, e.g. aufs, overlay, vfs") exgroup = migrate19_parser.add_mutually_exclusive_group() exgroup.add_argument( @@ -74,6 +82,9 @@ def main(arguments=None): if args.tempdir: tempfile.tempdir = args.tempdir + if args.verbose: + logger.setLevel(logging.DEBUG) + only_migrate = None if args.infile: only_migrate = set() @@ -115,6 +126,7 @@ def main(arguments=None): need_migrate = {} biggest = 0 + biggest_pdh = None for img in old_images: i = uuid_to_collection[img["collection"]] pdh = i["portable_data_hash"] @@ -123,17 +135,34 @@ def main(arguments=None): with CollectionReader(i["manifest_text"]) as c: if c.values()[0].size() > biggest: biggest = c.values()[0].size() + biggest_pdh = pdh + + + if args.storage_driver == "vfs": + will_need = (biggest*20) + else: + will_need = (biggest*2.5) if args.print_unmigrated: only_migrate = set() for pdh in need_migrate: - print pdh + print(pdh) return logger.info("Already migrated %i images", len(already_migrated)) logger.info("Need to migrate %i images", len(need_migrate)) logger.info("Using tempdir %s", tempfile.gettempdir()) - logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20)) + logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20)) + + df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()]) + ln = df_out.splitlines()[1] + filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1) + if int(available) <= will_need: + logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20)) + if not args.force: + exit(1) + else: + logger.warn("--force provided, will migrate anyway") if args.dry_run: return @@ -169,23 +198,51 @@ def main(arguments=None): "--env-file", envfile.name, "--volume", "%s:/var/lib/docker" % varlibdocker, "--volume", "%s:/root/.cache/arvados/docker" % dockercache, - "arvados/migrate-docker19", + "arvados/migrate-docker19:1.0", "/root/migrate.sh", "%s/%s" % (old_image["collection"], tarfile), tarfile[0:40], old_image["repo"], old_image["tag"], - uuid_to_collection[old_image["collection"]]["owner_uuid"]] + uuid_to_collection[old_image["collection"]]["owner_uuid"], + args.storage_driver] proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() + initial_space = re.search(r"Initial available space is (\d+)", out) + imgload_space = re.search(r"Available space after image load is (\d+)", out) + imgupgrade_space = re.search(r"Available space after image upgrade is (\d+)", out) + keepdocker_space = re.search(r"Available space after arv-keepdocker is (\d+)", out) + cleanup_space = re.search(r"Available space after cleanup is (\d+)", out) + + if initial_space: + isp = int(initial_space.group(1)) + logger.debug("Available space initially: %i MiB", (isp)/(2**20)) + if imgload_space: + sp = int(imgload_space.group(1)) + logger.debug("Used after load: %i MiB", (isp-sp)/(2**20)) + if imgupgrade_space: + sp = int(imgupgrade_space.group(1)) + logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20)) + if keepdocker_space: + sp = int(keepdocker_space.group(1)) + logger.debug("Used after upload: %i MiB", (isp-sp)/(2**20)) + + if cleanup_space: + sp = int(cleanup_space.group(1)) + logger.info("Available after cleanup: %i MiB", (sp)/(2**20)) + if proc.returncode != 0: logger.error("Failed with return code %i", proc.returncode) logger.error("--- Stdout ---\n%s", out) logger.error("--- Stderr ---\n%s", err) raise MigrationFailed() + if args.verbose: + logger.info("--- Stdout ---\n%s", out) + logger.info("--- Stderr ---\n%s", err) + migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out) if migrated: newcol = CollectionReader(migrated.group(1)) -- 2.30.2