1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
16 "git.arvados.org/arvados.git/sdk/go/arvados"
17 "golang.org/x/net/context"
20 type singularityExecutor struct {
21 logf func(string, ...interface{})
25 imageFilename string // "sif" image
26 containerClient *arvados.Client
27 container arvados.Container
30 func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExecutor, error) {
31 tmpdir, err := ioutil.TempDir("", "crunch-run-singularity-")
35 return &singularityExecutor{
41 func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, create bool) (*arvados.Group, error) {
42 var gp arvados.GroupList
43 err := e.containerClient.RequestAndDecode(&gp,
44 arvados.EndpointGroupList.Method,
45 arvados.EndpointGroupList.Path,
46 nil, arvados.ListOptions{Filters: []arvados.Filter{
47 arvados.Filter{"owner_uuid", "=", ownerUuid},
48 arvados.Filter{"name", "=", name},
49 arvados.Filter{"group_class", "=", "project"},
54 if len(gp.Items) > 0 {
55 return &gp.Items[0], nil
60 var rgroup arvados.Group
61 err = e.containerClient.RequestAndDecode(&rgroup,
62 arvados.EndpointGroupCreate.Method,
63 arvados.EndpointGroupCreate.Path,
64 nil, map[string]interface{}{
65 "group": map[string]string{
66 "owner_uuid": ownerUuid,
68 "group_class": "project",
77 func (e *singularityExecutor) ImageLoaded(string) bool {
78 // Check if docker image is cached in keep & if so set imageFilename
83 // LoadImage will satisfy ContainerExecuter interface transforming
84 // containerImage into a sif file for later use.
85 func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
86 if e.imageFilename != "" {
87 // was set by ImageLoaded
91 e.logf("building singularity image")
92 // "singularity build" does not accept a
93 // docker-archive://... filename containing a ":" character,
94 // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
95 // symlink that doesn't have ":" chars.
96 err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
100 e.imageFilename = e.tmpdir + "/image.sif"
101 build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
102 e.logf("%v", build.Args)
103 out, err := build.CombinedOutput()
104 // INFO: Starting build...
105 // Getting image source signatures
106 // Copying blob ab15617702de done
107 // Copying config 651e02b8a2 done
108 // Writing manifest to image destination
109 // Storing signatures
110 // 2021/04/22 14:42:14 info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
111 // INFO: Creating SIF file...
112 // INFO: Build complete: arvados-jobs.latest.sif
118 // Cache the image to keep
119 cacheGroup, err := e.getOrCreateProject(e.container.RuntimeUserUUID, ".cache", true)
121 e.logf("error getting '.cache' project: %s", err)
124 imageGroup, err := e.getOrCreateProject(cacheGroup.UUID, "auto-generated singularity images", true)
126 e.logf("error getting 'auto-generated singularity images' project: %s", err)
130 parts := strings.Split(imageTarballPath, "/")
131 imageId := parts[len(parts)-1]
133 var imageCollection arvados.Collection
134 err = e.containerClient.RequestAndDecode(&imageCollection,
135 arvados.EndpointCollectionCreate.Method,
136 arvados.EndpointCollectionCreate.Path,
137 nil, map[string]interface{}{
138 "collection": map[string]string{
139 "owner_uuid": imageGroup.UUID,
140 "name": fmt.Sprintf("singularity image for %s", imageId),
144 e.logf("error creating 'auto-generated singularity images' collection: %s", err)
150 func (e *singularityExecutor) Create(spec containerSpec) error {
155 func (e *singularityExecutor) Start() error {
156 args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
157 if !e.spec.EnableNetwork {
158 args = append(args, "--net", "--network=none")
160 readonlyflag := map[bool]string{
165 for path, _ := range e.spec.BindMounts {
166 binds = append(binds, path)
169 for _, path := range binds {
170 mount := e.spec.BindMounts[path]
171 args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
173 args = append(args, e.imageFilename)
174 args = append(args, e.spec.Command...)
176 // This is for singularity 3.5.2. There are some behaviors
177 // that will change in singularity 3.6, please see:
178 // https://sylabs.io/guides/3.7/user-guide/environment_and_metadata.html
179 // https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
180 env := make([]string, 0, len(e.spec.Env))
181 for k, v := range e.spec.Env {
182 env = append(env, "SINGULARITYENV_"+k+"="+v)
185 path, err := exec.LookPath(args[0])
194 Stdout: e.spec.Stdout,
195 Stderr: e.spec.Stderr,
205 func (e *singularityExecutor) CgroupID() string {
209 func (e *singularityExecutor) Stop() error {
210 if err := e.child.Process.Signal(syscall.Signal(0)); err != nil {
211 // process already exited
214 return e.child.Process.Signal(syscall.SIGKILL)
217 func (e *singularityExecutor) Wait(context.Context) (int, error) {
218 err := e.child.Wait()
219 if err, ok := err.(*exec.ExitError); ok {
220 return err.ProcessState.ExitCode(), nil
225 return e.child.ProcessState.ExitCode(), nil
228 func (e *singularityExecutor) Close() {
229 err := os.RemoveAll(e.tmpdir)
231 e.logf("error removing temp dir: %s", err)
235 func (e *singularityExecutor) SetArvadoClient(containerClient *arvados.Client, container arvados.Container) {
236 e.containerClient = containerClient
237 e.container = container