X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6fe5fe41128cfe4fc9b5b6f6c1f092c4d01c4e4e..540b72d62a94015f116ba077e279a5f10d666778:/sdk/python/arvados/commands/keepdocker.py diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py index 537ea3a945..922256a27e 100644 --- a/sdk/python/arvados/commands/keepdocker.py +++ b/sdk/python/arvados/commands/keepdocker.py @@ -85,7 +85,8 @@ class DockerError(Exception): def popen_docker(cmd, *args, **kwargs): manage_stdin = ('stdin' not in kwargs) kwargs.setdefault('stdin', subprocess.PIPE) - kwargs.setdefault('stdout', sys.stderr) + kwargs.setdefault('stdout', subprocess.PIPE) + kwargs.setdefault('stderr', subprocess.PIPE) try: docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs) except OSError: # No docker in $PATH, try docker.io @@ -257,7 +258,7 @@ def _new_image_listing(link, dockerhash, repo='', tag=''): 'tag': tag, } -def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None): +def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None, project_uuid=None): """List all Docker images known to the api_client with image_name and image_tag. If no image_name is given, defaults to listing all Docker images. @@ -272,13 +273,18 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) search_filters = [] repo_links = None hash_links = None + + project_filter = [] + if project_uuid is not None: + project_filter = [["owner_uuid", "=", project_uuid]] + if image_name: # Find images with the name the user specified. search_links = _get_docker_links( api_client, num_retries, filters=[['link_class', '=', 'docker_image_repo+tag'], ['name', '=', - '{}:{}'.format(image_name, image_tag or 'latest')]]) + '{}:{}'.format(image_name, image_tag or 'latest')]]+project_filter) if search_links: repo_links = search_links else: @@ -286,7 +292,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) search_links = _get_docker_links( api_client, num_retries, filters=[['link_class', '=', 'docker_image_hash'], - ['name', 'ilike', image_name + '%']]) + ['name', 'ilike', image_name + '%']]+project_filter) hash_links = search_links # Only list information about images that were found in the search. search_filters.append(['head_uuid', 'in', @@ -298,7 +304,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) if hash_links is None: hash_links = _get_docker_links( api_client, num_retries, - filters=search_filters + [['link_class', '=', 'docker_image_hash']]) + filters=search_filters + [['link_class', '=', 'docker_image_hash']]+project_filter) hash_link_map = {link['head_uuid']: link for link in reversed(hash_links)} # Each collection may have more than one name (though again, one name @@ -308,7 +314,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) repo_links = _get_docker_links( api_client, num_retries, filters=search_filters + [['link_class', '=', - 'docker_image_repo+tag']]) + 'docker_image_repo+tag']]+project_filter) seen_image_names = collections.defaultdict(set) images = [] for link in repo_links: @@ -320,7 +326,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) dockerhash = hash_link_map[collection_uuid]['name'] except KeyError: dockerhash = '' - name_parts = link['name'].split(':', 1) + name_parts = link['name'].rsplit(':', 1) images.append(_new_image_listing(link, dockerhash, *name_parts)) # Find any image hash links that did not have a corresponding name link, @@ -336,7 +342,7 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None) # Remove any image listings that refer to unknown collections. existing_coll_uuids = {coll['uuid'] for coll in arvados.util.list_all( api_client.collections().list, num_retries, - filters=[['uuid', 'in', [im['collection'] for im in images]]], + filters=[['uuid', 'in', [im['collection'] for im in images]]]+project_filter, select=['uuid'])} return [(image['collection'], image) for image in images if image['collection'] in existing_coll_uuids] @@ -353,7 +359,7 @@ def _uuid2pdh(api, uuid): def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None): args = arg_parser.parse_args(arguments) if api is None: - api = arvados.api('v1') + api = arvados.api('v1', num_retries=args.retries) if args.image is None or args.image == 'images': fmt = "{:30} {:10} {:12} {:29} {:20}\n" @@ -380,23 +386,40 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None) elif args.tag is None: args.tag = 'latest' + if '/' in args.image: + hostport, path = args.image.split('/', 1) + if hostport.endswith(':443'): + # "docker pull host:443/asdf" transparently removes the + # :443 (which is redundant because https is implied) and + # after it succeeds "docker images" will list "host/asdf", + # not "host:443/asdf". If we strip the :443 then the name + # doesn't change underneath us. + args.image = '/'.join([hostport[:-4], path]) + # Pull the image if requested, unless the image is specified as a hash # that we already have. if args.pull and not find_image_hashes(args.image): pull_image(args.image, args.tag) + images_in_arv = list_images_in_arv(api, args.retries, args.image, args.tag) + + image_hash = None try: image_hash = find_one_image_hash(args.image, args.tag) + if not docker_image_compatible(api, image_hash): + if args.force_image_format: + logger.warning("forcing incompatible image") + else: + logger.error("refusing to store " \ + "incompatible format (use --force-image-format to override)") + sys.exit(1) except DockerError as error: - logger.error(str(error)) - sys.exit(1) - - if not docker_image_compatible(api, image_hash): - if args.force_image_format: - logger.warning("forcing incompatible image") + if images_in_arv: + # We don't have Docker / we don't have the image locally, + # use image that's already uploaded to Arvados + image_hash = images_in_arv[0][1]['dockerhash'] else: - logger.error("refusing to store " \ - "incompatible format (use --force-image-format to override)") + logger.error(str(error)) sys.exit(1) image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None