<notextile>
<pre><code># <span class="userinput">apt-get --no-install-recommends install curl gnupg2</span>
-# <span class="userinput">curl https://apt.arvados.org/pubkey.gpg -o /etc/apt/trusted.gpg.d/arvados.asc</span>
+# <span class="userinput">curl -s https://apt.arvados.org/pubkey.gpg -o /etc/apt/trusted.gpg.d/arvados.asc</span>
</code></pre>
</notextile>
** The Docker ENTRYPOINT instruction is ignored.
* Arvados is tested with Singularity version 3.7.4. Other versions may not work.
+*Notes*:
+
+* Docker images are converted on the fly by @mksquashfs@, which can consume a considerable amount of RAM. The RAM usage of mksquashfs can be restricted in @/etc/singularity/singularity.conf@ with a line like @mksquashfs mem = 512M@. The amount of memory made available for mksquashfs should be configured lower than the smallest amount of memory requested by a container on the cluster to avoid the conversion being killed for using too much memory.
+
h2(#configuration). Configuration
To use singularity, first make sure "Singularity is installed":https://sylabs.io/guides/3.7/user-guide/quick_start.html on your cloud worker image or SLURM/LSF compute nodes as applicable. Note @squashfs-tools@ is required.
To use the Python SDK elsewhere, you can install from PyPI or a distribution package.
-As of Arvados 2.1, the Python SDK requires Python 3.5+. The last version to support Python 2.7 is Arvados 2.0.4.
+As of Arvados 2.2, the Python SDK requires Python 3.6+. The last version to support Python 2.7 is Arvados 2.0.4.
h2. Option 1: Install from a distribution package
return err
}
+ // Set up a cache and tmp dir for singularity build
+ err = os.Mkdir(e.tmpdir+"/cache", 0700)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(e.tmpdir + "/cache")
+ err = os.Mkdir(e.tmpdir+"/tmp", 0700)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(e.tmpdir + "/tmp")
+
build := exec.Command("singularity", "build", imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
+ build.Env = os.Environ()
+ build.Env = append(build.Env, "SINGULARITY_CACHEDIR="+e.tmpdir+"/cache")
+ build.Env = append(build.Env, "SINGULARITY_TMPDIR="+e.tmpdir+"/tmp")
e.logf("%v", build.Args)
out, err := build.CombinedOutput()
// INFO: Starting build...
end
-subcommands = %w(copy create edit get keep pipeline run tag ws)
+subcommands = %w(copy create edit get keep tag ws)
def exec_bin bin, opts
bin_path = `which #{bin.shellescape}`.strip
arv_edit client, arvados, global_opts, remaining_opts
when 'get'
arv_get client, arvados, global_opts, remaining_opts
- when 'copy', 'tag', 'ws', 'run'
+ when 'copy', 'tag', 'ws'
exec_bin "arv-#{subcommand}", remaining_opts
when 'keep'
@sub = remaining_opts.shift
elif isinstance(pattern, dict):
specs.append(pattern)
elif isinstance(pattern, str):
- specs.append({"pattern": pattern, "required": sf.get("required")})
+ if builder.cwlVersion == "v1.0":
+ specs.append({"pattern": pattern, "required": True})
+ else:
+ specs.append({"pattern": pattern, "required": sf.get("required")})
else:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Expression must return list, object, string or null")
if isinstance(sf, dict):
if sf.get("class") == "File":
pattern = None
+ if sf.get("location") is None:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "File object is missing 'location': %s" % sf)
sfpath = sf["location"]
required = True
else:
kwargs.setdefault('stdin', subprocess.PIPE)
kwargs.setdefault('stdout', sys.stderr)
try:
- docker_proc = subprocess.Popen(['docker.io'] + cmd, *args, **kwargs)
- except OSError: # No docker.io in $PATH
docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs)
+ except OSError: # No docker in $PATH, try docker.io
+ docker_proc = subprocess.Popen(['docker.io'] + cmd, *args, **kwargs)
if manage_stdin:
docker_proc.stdin.close()
return docker_proc
check_docker(list_proc, "images")
def find_image_hashes(image_search, image_tag=None):
- # Given one argument, search for Docker images with matching hashes,
- # and return their full hashes in a set.
- # Given two arguments, also search for a Docker image with the
- # same repository and tag. If one is found, return its hash in a
- # set; otherwise, fall back to the one-argument hash search.
- # Returns None if no match is found, or a hash search is ambiguous.
- hash_search = image_search.lower()
- hash_matches = set()
- for image in docker_images():
- if (image.repo == image_search) and (image.tag == image_tag):
- return set([image.hash])
- elif image.hash.startswith(hash_search):
- hash_matches.add(image.hash)
- return hash_matches
+ # Query for a Docker images with the repository and tag and return
+ # the image ids in a list. Returns empty list if no match is
+ # found.
+
+ list_proc = popen_docker(['inspect', "%s%s" % (image_search, ":"+image_tag if image_tag else "")], stdout=subprocess.PIPE)
+
+ inspect = list_proc.stdout.read()
+ list_proc.stdout.close()
+
+ imageinfo = json.loads(inspect)
+
+ return [i["Id"] for i in imageinfo]
def find_one_image_hash(image_search, image_tag=None):
hashes = find_image_hashes(image_search, image_tag)
projects / arvados.git / commitdiff