From 728fbdbae7d8e926f64a09d3f20aad6bdb67435e Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Tue, 10 Jun 2014 16:34:01 -0400 Subject: [PATCH] 2879: Add arv-keepdocker command. This puts a Docker image in Keep, and makes Arvados links to help find the Collection by the image's names. --- sdk/cli/bin/arv | 5 +- sdk/python/arvados/commands/keepdocker.py | 204 ++++++++++++++++++++++ sdk/python/bin/arv-keepdocker | 4 + sdk/python/setup.py | 3 +- 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 sdk/python/arvados/commands/keepdocker.py create mode 100755 sdk/python/bin/arv-keepdocker diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv index 31cbeec70c..b485b7b10f 100755 --- a/sdk/cli/bin/arv +++ b/sdk/cli/bin/arv @@ -42,13 +42,16 @@ when 'keep' elsif ['less', 'check'].index @sub then # wh* shims exec `which wh#{@sub}`.strip, *ARGV + elsif @sub == 'docker' + exec `which arv-keepdocker`.strip, *ARGV else puts "Usage: \n" + "#{$0} keep ls\n" + "#{$0} keep get\n" + "#{$0} keep put\n" + "#{$0} keep less\n" + - "#{$0} keep check\n" + "#{$0} keep check\n" + + "#{$0} keep docker\n" end abort when 'pipeline' diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py new file mode 100644 index 0000000000..0c4930e039 --- /dev/null +++ b/sdk/python/arvados/commands/keepdocker.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python + +import argparse +import errno +import json +import os +import subprocess +import sys +import tarfile +import tempfile + +from collections import namedtuple +from stat import * + +import arvados +import arvados.commands._util as arv_cmd +import arvados.commands.put as arv_put + +STAT_CACHE_ERRORS = (IOError, OSError, ValueError) + +DockerImage = namedtuple('DockerImage', + ['repo', 'tag', 'hash', 'created', 'vsize']) + +opt_parser = argparse.ArgumentParser(add_help=False) +opt_parser.add_argument( + '-f', '--force', action='store_true', default=False, + help="Re-upload the image even if it already exists on the server") + +_group = opt_parser.add_mutually_exclusive_group() +_group.add_argument( + '--pull', action='store_true', default=True, + help="Pull the latest image from Docker repositories first (default)") +_group.add_argument( + '--no-pull', action='store_false', dest='pull', + help="Don't pull images from Docker repositories") + +opt_parser.add_argument( + 'image', + help="Docker image to upload, as a repository name or hash") +opt_parser.add_argument( + 'tag', nargs='?', default='latest', + help="Tag of the Docker image to upload (default 'latest')") + +arg_parser = argparse.ArgumentParser( + description="Upload a Docker image to Arvados", + parents=[opt_parser, arv_put.run_opts]) + +class DockerError(Exception): + pass + + +def popen_docker(cmd, *args, **kwargs): + manage_stdin = ('stdin' not in kwargs) + kwargs.setdefault('stdin', subprocess.PIPE) + kwargs.setdefault('stdout', sys.stderr) + try: + docker_proc = subprocess.Popen(['docker.io'] + cmd, *args, **kwargs) + except OSError: # No docker.io in $PATH + docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs) + if manage_stdin: + docker_proc.stdin.close() + return docker_proc + +def check_docker(proc, description): + proc.wait() + if proc.returncode != 0: + raise DockerError("docker {} returned status code {}". + format(description, proc.returncode)) + +def docker_images(): + # Yield a DockerImage tuple for each installed image. + list_proc = popen_docker(['images', '--no-trunc'], stdout=subprocess.PIPE) + list_output = iter(list_proc.stdout) + next(list_output) # Ignore the header line + for line in list_output: + words = line.split() + size_index = len(words) - 2 + repo, tag, imageid = words[:3] + ctime = ' '.join(words[3:size_index]) + vsize = ' '.join(words[size_index:]) + yield DockerImage(repo, tag, imageid, ctime, vsize) + list_proc.stdout.close() + check_docker(list_proc, "images") + +def find_image_hash(image_name, image_tag): + hash_search = image_name.lower() + hash_matches = set() + for image in docker_images(): + if (image.repo == image_name) and (image.tag == image_tag): + return image.hash + elif image.hash.startswith(hash_search): + hash_matches.add(image.hash) + if len(hash_matches) == 1: + return hash_matches.pop() + return None + +def stat_cache_name(image_file): + return getattr(image_file, 'name', image_file) + '.stat' + +def pull_image(image_name, image_tag): + check_docker(popen_docker(['pull', '-t', image_tag, image_name]), "pull") + +def save_image(image_hash, image_file): + # Save the specified Docker image to image_file, then try to save its + # stats so we can try to resume after interruption. + check_docker(popen_docker(['save', image_hash], stdout=image_file), + "save") + image_file.flush() + try: + with open(stat_cache_name(image_file), 'w') as statfile: + json.dump(tuple(os.fstat(image_file.fileno())), statfile) + except STAT_CACHE_ERRORS: + pass # We won't resume from this cache. No big deal. + +def prep_image_file(filename): + # Return a file object ready to save a Docker image, + # and a boolean indicating whether or not we need to actually save the + # image (False if a cached save is available). + cache_dir = arv_cmd.make_home_conf_dir( + os.path.join('.cache', 'arvados', 'docker'), 0o700) + if cache_dir is None: + image_file = tempfile.NamedTemporaryFile(suffix='.tar') + need_save = True + else: + file_path = os.path.join(cache_dir, filename) + try: + with open(stat_cache_name(file_path)) as statfile: + prev_stat = json.load(statfile) + now_stat = os.stat(file_path) + need_save = any(prev_stat[field] != now_stat[field] + for field in [ST_MTIME, ST_SIZE]) + except STAT_CACHE_ERRORS + (AttributeError, IndexError): + need_save = True # We couldn't compare against old stats + image_file = open(file_path, 'w+b' if need_save else 'rb') + return image_file, need_save + +def make_link(link_class, link_name, **link_attrs): + link_attrs.update({'link_class': link_class, 'name': link_name}) + return arvados.api('v1').links().create(body=link_attrs).execute() + +def main(arguments=None): + args = arg_parser.parse_args(arguments) + + # Pull the image if requested, unless the image is specified as a hash + # that we already have. + if args.pull and (find_image_hash(args.image, None) is None): + pull_image(args.image, args.tag) + + image_hash = find_image_hash(args.image, args.tag) + if image_hash is None: + print >>sys.stderr, "arv-keepdocker: No image found." + sys.exit(1) + elif not args.force: + # Abort if this image is already in Arvados. + existing_links = arvados.api('v1').links().list( + filters=[['link_class', '=', 'docker_image_hash'], + ['name', '=', image_hash]]).execute()['items'] + if existing_links: + message = [ + "arv-keepdocker: Image {} already stored in collection(s):". + format(image_hash)] + message.extend(link['head_uuid'] for link in existing_links) + print >>sys.stderr, "\n".join(message) + sys.exit(1) + + # Open a file for the saved image, and write it if needed. + outfile_name = '{}.tar'.format(image_hash) + image_file, need_save = prep_image_file(outfile_name) + if need_save: + save_image(image_hash, image_file) + + # Call arv-put with switches we inherited from it + # (a.k.a., switches that aren't our own). + put_args = opt_parser.parse_known_args(arguments)[1] + coll_uuid = arv_put.main( + put_args + ['--filename', outfile_name, image_file.name]).strip() + + # Read the image metadata and make Arvados links from it. + image_file.seek(0) + image_tar = tarfile.open(fileobj=image_file) + json_file = image_tar.extractfile(image_tar.getmember(image_hash + '/json')) + image_metadata = json.load(json_file) + json_file.close() + image_tar.close() + link_base = {'head_uuid': coll_uuid, 'properties': {}} + if 'created' in image_metadata: + link_base['properties']['image_timestamp'] = image_metadata['created'] + + make_link('docker_image_hash', image_hash, **link_base) + if not image_hash.startswith(args.image.lower()): + make_link('docker_image_repository', args.image, **link_base) + make_link('docker_image_tag', args.tag, **link_base) + + # Clean up. + image_file.close() + for filename in [stat_cache_name(image_file), image_file.name]: + try: + os.unlink(filename) + except OSError as error: + if error.errno != errno.ENOENT: + raise + +if __name__ == '__main__': + main() diff --git a/sdk/python/bin/arv-keepdocker b/sdk/python/bin/arv-keepdocker new file mode 100755 index 0000000000..20d9d62b72 --- /dev/null +++ b/sdk/python/bin/arv-keepdocker @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +from arvados.commands.keepdocker import main +main() diff --git a/sdk/python/setup.py b/sdk/python/setup.py index ec89977bf0..a2098630f9 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -11,9 +11,10 @@ setup(name='arvados-python-client', packages=find_packages(), scripts=[ 'bin/arv-get', - 'bin/arv-put', + 'bin/arv-keepdocker', 'bin/arv-ls', 'bin/arv-normalize', + 'bin/arv-put', ], install_requires=[ 'python-gflags', -- 2.30.2