12 from collections import namedtuple
16 import arvados.commands._util as arv_cmd
17 import arvados.commands.put as arv_put
19 STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
21 DockerImage = namedtuple('DockerImage',
22 ['repo', 'tag', 'hash', 'created', 'vsize'])
24 opt_parser = argparse.ArgumentParser(add_help=False)
25 opt_parser.add_argument(
26 '-f', '--force', action='store_true', default=False,
27 help="Re-upload the image even if it already exists on the server")
29 _group = opt_parser.add_mutually_exclusive_group()
31 '--pull', action='store_true', default=True,
32 help="Pull the latest image from Docker repositories first (default)")
34 '--no-pull', action='store_false', dest='pull',
35 help="Don't pull images from Docker repositories")
37 opt_parser.add_argument(
39 help="Docker image to upload, as a repository name or hash")
40 opt_parser.add_argument(
41 'tag', nargs='?', default='latest',
42 help="Tag of the Docker image to upload (default 'latest')")
44 arg_parser = argparse.ArgumentParser(
45 description="Upload a Docker image to Arvados",
46 parents=[opt_parser, arv_put.run_opts])
48 class DockerError(Exception):
52 def popen_docker(cmd, *args, **kwargs):
53 manage_stdin = ('stdin' not in kwargs)
54 kwargs.setdefault('stdin', subprocess.PIPE)
55 kwargs.setdefault('stdout', sys.stderr)
57 docker_proc = subprocess.Popen(['docker.io'] + cmd, *args, **kwargs)
58 except OSError: # No docker.io in $PATH
59 docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs)
61 docker_proc.stdin.close()
64 def check_docker(proc, description):
66 if proc.returncode != 0:
67 raise DockerError("docker {} returned status code {}".
68 format(description, proc.returncode))
71 # Yield a DockerImage tuple for each installed image.
72 list_proc = popen_docker(['images', '--no-trunc'], stdout=subprocess.PIPE)
73 list_output = iter(list_proc.stdout)
74 next(list_output) # Ignore the header line
75 for line in list_output:
77 size_index = len(words) - 2
78 repo, tag, imageid = words[:3]
79 ctime = ' '.join(words[3:size_index])
80 vsize = ' '.join(words[size_index:])
81 yield DockerImage(repo, tag, imageid, ctime, vsize)
82 list_proc.stdout.close()
83 check_docker(list_proc, "images")
85 def find_image_hash(image_name, image_tag):
86 hash_search = image_name.lower()
88 for image in docker_images():
89 if (image.repo == image_name) and (image.tag == image_tag):
91 elif image.hash.startswith(hash_search):
92 hash_matches.add(image.hash)
93 if len(hash_matches) == 1:
94 return hash_matches.pop()
97 def stat_cache_name(image_file):
98 return getattr(image_file, 'name', image_file) + '.stat'
100 def pull_image(image_name, image_tag):
101 check_docker(popen_docker(['pull', '-t', image_tag, image_name]), "pull")
103 def save_image(image_hash, image_file):
104 # Save the specified Docker image to image_file, then try to save its
105 # stats so we can try to resume after interruption.
106 check_docker(popen_docker(['save', image_hash], stdout=image_file),
110 with open(stat_cache_name(image_file), 'w') as statfile:
111 json.dump(tuple(os.fstat(image_file.fileno())), statfile)
112 except STAT_CACHE_ERRORS:
113 pass # We won't resume from this cache. No big deal.
115 def prep_image_file(filename):
116 # Return a file object ready to save a Docker image,
117 # and a boolean indicating whether or not we need to actually save the
118 # image (False if a cached save is available).
119 cache_dir = arv_cmd.make_home_conf_dir(
120 os.path.join('.cache', 'arvados', 'docker'), 0o700)
121 if cache_dir is None:
122 image_file = tempfile.NamedTemporaryFile(suffix='.tar')
125 file_path = os.path.join(cache_dir, filename)
127 with open(stat_cache_name(file_path)) as statfile:
128 prev_stat = json.load(statfile)
129 now_stat = os.stat(file_path)
130 need_save = any(prev_stat[field] != now_stat[field]
131 for field in [ST_MTIME, ST_SIZE])
132 except STAT_CACHE_ERRORS + (AttributeError, IndexError):
133 need_save = True # We couldn't compare against old stats
134 image_file = open(file_path, 'w+b' if need_save else 'rb')
135 return image_file, need_save
137 def make_link(link_class, link_name, **link_attrs):
138 link_attrs.update({'link_class': link_class, 'name': link_name})
139 return arvados.api('v1').links().create(body=link_attrs).execute()
141 def main(arguments=None):
142 args = arg_parser.parse_args(arguments)
144 # Pull the image if requested, unless the image is specified as a hash
145 # that we already have.
146 if args.pull and (find_image_hash(args.image, None) is None):
147 pull_image(args.image, args.tag)
149 image_hash = find_image_hash(args.image, args.tag)
150 if image_hash is None:
151 print >>sys.stderr, "arv-keepdocker: No image found."
154 # Abort if this image is already in Arvados.
155 existing_links = arvados.api('v1').links().list(
156 filters=[['link_class', '=', 'docker_image_hash'],
157 ['name', '=', image_hash]]).execute()['items']
160 "arv-keepdocker: Image {} already stored in collection(s):".
162 message.extend(link['head_uuid'] for link in existing_links)
163 print >>sys.stderr, "\n".join(message)
166 # Open a file for the saved image, and write it if needed.
167 outfile_name = '{}.tar'.format(image_hash)
168 image_file, need_save = prep_image_file(outfile_name)
170 save_image(image_hash, image_file)
172 # Call arv-put with switches we inherited from it
173 # (a.k.a., switches that aren't our own).
174 put_args = opt_parser.parse_known_args(arguments)[1]
175 coll_uuid = arv_put.main(
176 put_args + ['--filename', outfile_name, image_file.name]).strip()
178 # Read the image metadata and make Arvados links from it.
180 image_tar = tarfile.open(fileobj=image_file)
181 json_file = image_tar.extractfile(image_tar.getmember(image_hash + '/json'))
182 image_metadata = json.load(json_file)
185 link_base = {'head_uuid': coll_uuid, 'properties': {}}
186 if 'created' in image_metadata:
187 link_base['properties']['image_timestamp'] = image_metadata['created']
189 make_link('docker_image_hash', image_hash, **link_base)
190 if not image_hash.startswith(args.image.lower()):
191 make_link('docker_image_repository', args.image, **link_base)
192 make_link('docker_image_tag', args.tag, **link_base)
196 for filename in [stat_cache_name(image_file), image_file.name]:
199 except OSError as error:
200 if error.errno != errno.ENOENT:
203 if __name__ == '__main__':