18238: Another try at Singularity $HOME fix
[arvados.git] / lib / crunchrun / singularity.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package crunchrun
6
7 import (
8         "fmt"
9         "io/ioutil"
10         "os"
11         "os/exec"
12         "sort"
13         "syscall"
14         "time"
15
16         "git.arvados.org/arvados.git/sdk/go/arvados"
17         "golang.org/x/net/context"
18 )
19
20 type singularityExecutor struct {
21         logf          func(string, ...interface{})
22         spec          containerSpec
23         tmpdir        string
24         child         *exec.Cmd
25         imageFilename string // "sif" image
26 }
27
28 func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExecutor, error) {
29         tmpdir, err := ioutil.TempDir("", "crunch-run-singularity-")
30         if err != nil {
31                 return nil, err
32         }
33         return &singularityExecutor{
34                 logf:   logf,
35                 tmpdir: tmpdir,
36         }, nil
37 }
38
39 func (e *singularityExecutor) Runtime() string { return "singularity" }
40
41 func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) {
42         var gp arvados.GroupList
43         err := containerClient.RequestAndDecode(&gp,
44                 arvados.EndpointGroupList.Method,
45                 arvados.EndpointGroupList.Path,
46                 nil, arvados.ListOptions{Filters: []arvados.Filter{
47                         arvados.Filter{"owner_uuid", "=", ownerUuid},
48                         arvados.Filter{"name", "=", name},
49                         arvados.Filter{"group_class", "=", "project"},
50                 },
51                         Limit: 1})
52         if err != nil {
53                 return nil, err
54         }
55         if len(gp.Items) == 1 {
56                 return &gp.Items[0], nil
57         }
58
59         var rgroup arvados.Group
60         err = containerClient.RequestAndDecode(&rgroup,
61                 arvados.EndpointGroupCreate.Method,
62                 arvados.EndpointGroupCreate.Path,
63                 nil, map[string]interface{}{
64                         "group": map[string]string{
65                                 "owner_uuid":  ownerUuid,
66                                 "name":        name,
67                                 "group_class": "project",
68                         },
69                 })
70         if err != nil {
71                 return nil, err
72         }
73         return &rgroup, nil
74 }
75
76 func (e *singularityExecutor) checkImageCache(dockerImageID string, container arvados.Container, arvMountPoint string,
77         containerClient *arvados.Client) (collection *arvados.Collection, err error) {
78
79         // Cache the image to keep
80         cacheGroup, err := e.getOrCreateProject(container.RuntimeUserUUID, ".cache", containerClient)
81         if err != nil {
82                 return nil, fmt.Errorf("error getting '.cache' project: %v", err)
83         }
84         imageGroup, err := e.getOrCreateProject(cacheGroup.UUID, "auto-generated singularity images", containerClient)
85         if err != nil {
86                 return nil, fmt.Errorf("error getting 'auto-generated singularity images' project: %s", err)
87         }
88
89         collectionName := fmt.Sprintf("singularity image for %v", dockerImageID)
90         var cl arvados.CollectionList
91         err = containerClient.RequestAndDecode(&cl,
92                 arvados.EndpointCollectionList.Method,
93                 arvados.EndpointCollectionList.Path,
94                 nil, arvados.ListOptions{Filters: []arvados.Filter{
95                         arvados.Filter{"owner_uuid", "=", imageGroup.UUID},
96                         arvados.Filter{"name", "=", collectionName},
97                 },
98                         Limit: 1})
99         if err != nil {
100                 return nil, fmt.Errorf("error querying for collection '%v': %v", collectionName, err)
101         }
102         var imageCollection arvados.Collection
103         if len(cl.Items) == 1 {
104                 imageCollection = cl.Items[0]
105         } else {
106                 collectionName := "converting " + collectionName
107                 exp := time.Now().Add(24 * 7 * 2 * time.Hour)
108                 err = containerClient.RequestAndDecode(&imageCollection,
109                         arvados.EndpointCollectionCreate.Method,
110                         arvados.EndpointCollectionCreate.Path,
111                         nil, map[string]interface{}{
112                                 "collection": map[string]string{
113                                         "owner_uuid": imageGroup.UUID,
114                                         "name":       collectionName,
115                                         "trash_at":   exp.UTC().Format(time.RFC3339),
116                                 },
117                                 "ensure_unique_name": true,
118                         })
119                 if err != nil {
120                         return nil, fmt.Errorf("error creating '%v' collection: %s", collectionName, err)
121                 }
122
123         }
124
125         return &imageCollection, nil
126 }
127
128 // LoadImage will satisfy ContainerExecuter interface transforming
129 // containerImage into a sif file for later use.
130 func (e *singularityExecutor) LoadImage(dockerImageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
131         containerClient *arvados.Client) error {
132
133         var imageFilename string
134         var sifCollection *arvados.Collection
135         var err error
136         if containerClient != nil {
137                 sifCollection, err = e.checkImageCache(dockerImageID, container, arvMountPoint, containerClient)
138                 if err != nil {
139                         return err
140                 }
141                 imageFilename = fmt.Sprintf("%s/by_uuid/%s/image.sif", arvMountPoint, sifCollection.UUID)
142         } else {
143                 imageFilename = e.tmpdir + "/image.sif"
144         }
145
146         if _, err := os.Stat(imageFilename); os.IsNotExist(err) {
147                 // Make sure the docker image is readable, and error
148                 // out if not.
149                 if _, err := os.Stat(imageTarballPath); err != nil {
150                         return err
151                 }
152
153                 e.logf("building singularity image")
154                 // "singularity build" does not accept a
155                 // docker-archive://... filename containing a ":" character,
156                 // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
157                 // symlink that doesn't have ":" chars.
158                 err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
159                 if err != nil {
160                         return err
161                 }
162
163                 build := exec.Command("singularity", "build", imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
164                 e.logf("%v", build.Args)
165                 out, err := build.CombinedOutput()
166                 // INFO:    Starting build...
167                 // Getting image source signatures
168                 // Copying blob ab15617702de done
169                 // Copying config 651e02b8a2 done
170                 // Writing manifest to image destination
171                 // Storing signatures
172                 // 2021/04/22 14:42:14  info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
173                 // INFO:    Creating SIF file...
174                 // INFO:    Build complete: arvados-jobs.latest.sif
175                 e.logf("%s", out)
176                 if err != nil {
177                         return err
178                 }
179         }
180
181         if containerClient == nil {
182                 e.imageFilename = imageFilename
183                 return nil
184         }
185
186         // update TTL to now + two weeks
187         exp := time.Now().Add(24 * 7 * 2 * time.Hour)
188
189         uuidPath, err := containerClient.PathForUUID("update", sifCollection.UUID)
190         if err != nil {
191                 e.logf("error PathForUUID: %v", err)
192                 return nil
193         }
194         var imageCollection arvados.Collection
195         err = containerClient.RequestAndDecode(&imageCollection,
196                 arvados.EndpointCollectionUpdate.Method,
197                 uuidPath,
198                 nil, map[string]interface{}{
199                         "collection": map[string]string{
200                                 "name":     fmt.Sprintf("singularity image for %v", dockerImageID),
201                                 "trash_at": exp.UTC().Format(time.RFC3339),
202                         },
203                 })
204         if err == nil {
205                 // If we just wrote the image to the cache, the
206                 // response also returns the updated PDH
207                 e.imageFilename = fmt.Sprintf("%s/by_id/%s/image.sif", arvMountPoint, imageCollection.PortableDataHash)
208                 return nil
209         }
210
211         e.logf("error updating/renaming collection for cached sif image: %v", err)
212         // Failed to update but maybe it lost a race and there is
213         // another cached collection in the same place, so check the cache
214         // again
215         sifCollection, err = e.checkImageCache(dockerImageID, container, arvMountPoint, containerClient)
216         if err != nil {
217                 return err
218         }
219         e.imageFilename = fmt.Sprintf("%s/by_id/%s/image.sif", arvMountPoint, sifCollection.PortableDataHash)
220
221         return nil
222 }
223
224 func (e *singularityExecutor) Create(spec containerSpec) error {
225         e.spec = spec
226         return nil
227 }
228
229 func (e *singularityExecutor) Start() error {
230         args := []string{"singularity", "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir}
231         if !e.spec.EnableNetwork {
232                 args = append(args, "--net", "--network=none")
233         }
234         readonlyflag := map[bool]string{
235                 false: "rw",
236                 true:  "ro",
237         }
238         var binds []string
239         for path, _ := range e.spec.BindMounts {
240                 binds = append(binds, path)
241         }
242         sort.Strings(binds)
243         for _, path := range binds {
244                 mount := e.spec.BindMounts[path]
245                 if path == e.spec.Env["HOME"] {
246                         // Singularity treates $HOME as special case
247                         args = append(args, "--home", mount.HostPath+":"+path)
248                 } else {
249                         args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
250                 }
251         }
252
253         // This is for singularity 3.5.2. There are some behaviors
254         // that will change in singularity 3.6, please see:
255         // https://sylabs.io/guides/3.7/user-guide/environment_and_metadata.html
256         // https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
257         env := make([]string, 0, len(e.spec.Env))
258         for k, v := range e.spec.Env {
259                 if k == "HOME" {
260                         // Singularity treates $HOME as special case, this is handled
261                         // with --home above
262                         continue
263                 }
264                 env = append(env, "SINGULARITYENV_"+k+"="+v)
265         }
266
267         args = append(args, e.imageFilename)
268         args = append(args, e.spec.Command...)
269
270         path, err := exec.LookPath(args[0])
271         if err != nil {
272                 return err
273         }
274         child := &exec.Cmd{
275                 Path:   path,
276                 Args:   args,
277                 Env:    env,
278                 Stdin:  e.spec.Stdin,
279                 Stdout: e.spec.Stdout,
280                 Stderr: e.spec.Stderr,
281         }
282         err = child.Start()
283         if err != nil {
284                 return err
285         }
286         e.child = child
287         return nil
288 }
289
290 func (e *singularityExecutor) CgroupID() string {
291         return ""
292 }
293
294 func (e *singularityExecutor) Stop() error {
295         if err := e.child.Process.Signal(syscall.Signal(0)); err != nil {
296                 // process already exited
297                 return nil
298         }
299         return e.child.Process.Signal(syscall.SIGKILL)
300 }
301
302 func (e *singularityExecutor) Wait(context.Context) (int, error) {
303         err := e.child.Wait()
304         if err, ok := err.(*exec.ExitError); ok {
305                 return err.ProcessState.ExitCode(), nil
306         }
307         if err != nil {
308                 return 0, err
309         }
310         return e.child.ProcessState.ExitCode(), nil
311 }
312
313 func (e *singularityExecutor) Close() {
314         err := os.RemoveAll(e.tmpdir)
315         if err != nil {
316                 e.logf("error removing temp dir: %s", err)
317         }
318 }