X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8fd331405028ebdbb97de58560057564aa530105..ad82cb7409dd62c3ac251e4f98d82ce774f6f528:/lib/crunchrun/singularity.go

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 4bec8c3ebe..f9ae073dc1 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -5,11 +5,15 @@
 package crunchrun
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"sort"
 	"syscall"
+	"time"
 
+	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"golang.org/x/net/context"
 )
 
@@ -32,39 +36,167 @@ func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExec
 	}, nil
 }
 
-func (e *singularityExecutor) ImageLoaded(string) bool {
-	return false
+func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) {
+	var gp arvados.GroupList
+	err := containerClient.RequestAndDecode(&gp,
+		arvados.EndpointGroupList.Method,
+		arvados.EndpointGroupList.Path,
+		nil, arvados.ListOptions{Filters: []arvados.Filter{
+			arvados.Filter{"owner_uuid", "=", ownerUuid},
+			arvados.Filter{"name", "=", name},
+			arvados.Filter{"group_class", "=", "project"},
+		},
+			Limit: 1})
+	if err != nil {
+		return nil, err
+	}
+	if len(gp.Items) == 1 {
+		return &gp.Items[0], nil
+	}
+
+	var rgroup arvados.Group
+	err = containerClient.RequestAndDecode(&rgroup,
+		arvados.EndpointGroupCreate.Method,
+		arvados.EndpointGroupCreate.Path,
+		nil, map[string]interface{}{
+			"group": map[string]string{
+				"owner_uuid":  ownerUuid,
+				"name":        name,
+				"group_class": "project",
+			},
+		})
+	if err != nil {
+		return nil, err
+	}
+	return &rgroup, nil
+}
+
+func (e *singularityExecutor) checkImageCache(dockerImageID string, container arvados.Container, arvMountPoint string,
+	containerClient *arvados.Client) (collectionUuid string, err error) {
+
+	// Cache the image to keep
+	cacheGroup, err := e.getOrCreateProject(container.RuntimeUserUUID, ".cache", containerClient)
+	if err != nil {
+		return "", fmt.Errorf("error getting '.cache' project: %v", err)
+	}
+	imageGroup, err := e.getOrCreateProject(cacheGroup.UUID, "auto-generated singularity images", containerClient)
+	if err != nil {
+		return "", fmt.Errorf("error getting 'auto-generated singularity images' project: %s", err)
+	}
+
+	collectionName := fmt.Sprintf("singularity image for %v", dockerImageID)
+	var cl arvados.CollectionList
+	err = containerClient.RequestAndDecode(&cl,
+		arvados.EndpointCollectionList.Method,
+		arvados.EndpointCollectionList.Path,
+		nil, arvados.ListOptions{Filters: []arvados.Filter{
+			arvados.Filter{"owner_uuid", "=", imageGroup.UUID},
+			arvados.Filter{"name", "=", collectionName},
+		},
+			Limit: 1})
+	if err != nil {
+		return "", fmt.Errorf("error querying for collection '%v': %v", collectionName, err)
+	}
+	var imageCollection arvados.Collection
+	if len(cl.Items) == 1 {
+		imageCollection = cl.Items[0]
+	} else {
+		collectionName := collectionName + " " + time.Now().UTC().Format(time.RFC3339)
+
+		err = containerClient.RequestAndDecode(&imageCollection,
+			arvados.EndpointCollectionCreate.Method,
+			arvados.EndpointCollectionCreate.Path,
+			nil, map[string]interface{}{
+				"collection": map[string]string{
+					"owner_uuid": imageGroup.UUID,
+					"name":       collectionName,
+				},
+			})
+		if err != nil {
+			e.logf("error creating '%v' collection: %s", collectionName, err)
+		}
+
+	}
+
+	return imageCollection.UUID, nil
 }
 
 // LoadImage will satisfy ContainerExecuter interface transforming
 // containerImage into a sif file for later use.
-func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
-	e.logf("building singularity image")
-	// "singularity build" does not accept a
-	// docker-archive://... filename containing a ":" character,
-	// as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
-	// symlink that doesn't have ":" chars.
-	err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+func (e *singularityExecutor) LoadImage(dockerImageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
+	containerClient *arvados.Client) error {
+
+	var sifCollectionUUID string
+	var imageFilename string
+	if containerClient != nil {
+		sifCollectionUUID, err := e.checkImageCache(dockerImageID, container, arvMountPoint, containerClient)
+		if err != nil {
+			return err
+		}
+		imageFilename = fmt.Sprintf("%s/by_uuid/%s/image.sif", arvMountPoint, sifCollectionUUID)
+	} else {
+		imageFilename = e.tmpdir + "/image.sif"
+	}
+
+	if _, err := os.Stat(imageFilename); os.IsNotExist(err) {
+		exec.Command("find", arvMountPoint+"/by_id/").Run()
+
+		e.logf("building singularity image")
+		// "singularity build" does not accept a
+		// docker-archive://... filename containing a ":" character,
+		// as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
+		// symlink that doesn't have ":" chars.
+		err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+		if err != nil {
+			return err
+		}
+
+		build := exec.Command("singularity", "build", imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
+		e.logf("%v", build.Args)
+		out, err := build.CombinedOutput()
+		// INFO:    Starting build...
+		// Getting image source signatures
+		// Copying blob ab15617702de done
+		// Copying config 651e02b8a2 done
+		// Writing manifest to image destination
+		// Storing signatures
+		// 2021/04/22 14:42:14  info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
+		// INFO:    Creating SIF file...
+		// INFO:    Build complete: arvados-jobs.latest.sif
+		e.logf("%s", out)
+		if err != nil {
+			return err
+		}
+	}
+
+	if containerClient == nil {
+		e.imageFilename = imageFilename
+		return nil
+	}
+
+	// update TTL to now + two weeks
+	exp := time.Now().Add(24 * 7 * 2 * time.Hour)
+
+	uuidPath, err := containerClient.PathForUUID("update", sifCollectionUUID)
 	if err != nil {
-		return err
+		e.logf("error PathForUUID: %v", err)
+		return nil
 	}
-	e.imageFilename = e.tmpdir + "/image.sif"
-	build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
-	e.logf("%v", build.Args)
-	out, err := build.CombinedOutput()
-	// INFO:    Starting build...
-	// Getting image source signatures
-	// Copying blob ab15617702de done
-	// Copying config 651e02b8a2 done
-	// Writing manifest to image destination
-	// Storing signatures
-	// 2021/04/22 14:42:14  info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
-	// INFO:    Creating SIF file...
-	// INFO:    Build complete: arvados-jobs.latest.sif
-	e.logf("%s", out)
+	var imageCollection arvados.Collection
+	err = containerClient.RequestAndDecode(&imageCollection,
+		arvados.EndpointCollectionUpdate.Method,
+		uuidPath,
+		nil, map[string]interface{}{
+			"collection": map[string]string{
+				"name":     fmt.Sprintf("singularity image for %v", dockerImageID),
+				"trash_at": exp.UTC().Format(time.RFC3339),
+			},
+		})
 	if err != nil {
-		return err
+		e.logf("error updating collection trash_at: %v", err)
 	}
+	e.imageFilename = fmt.Sprintf("%s/by_id/%s/image.sif", arvMountPoint, imageCollection.PortableDataHash)
+
 	return nil
 }
 
@@ -74,7 +206,7 @@ func (e *singularityExecutor) Create(spec containerSpec) error {
 }
 
 func (e *singularityExecutor) Start() error {
-	args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+	args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
 	if !e.spec.EnableNetwork {
 		args = append(args, "--net", "--network=none")
 	}
@@ -82,11 +214,15 @@ func (e *singularityExecutor) Start() error {
 		false: "rw",
 		true:  "ro",
 	}
-	for path, mount := range e.spec.BindMounts {
+	var binds []string
+	for path, _ := range e.spec.BindMounts {
+		binds = append(binds, path)
+	}
+	sort.Strings(binds)
+	for _, path := range binds {
+		mount := e.spec.BindMounts[path]
 		args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
 	}
-	args = append(args, e.imageFilename)
-	args = append(args, e.spec.Command...)
 
 	// This is for singularity 3.5.2. There are some behaviors
 	// that will change in singularity 3.6, please see:
@@ -94,9 +230,17 @@ func (e *singularityExecutor) Start() error {
 	// https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
 	env := make([]string, 0, len(e.spec.Env))
 	for k, v := range e.spec.Env {
-		env = append(env, "SINGULARITYENV_"+k+"="+v)
+		if k == "HOME" {
+			// $HOME is a special case
+			args = append(args, "--home="+v)
+		} else {
+			env = append(env, "SINGULARITYENV_"+k+"="+v)
+		}
 	}
 
+	args = append(args, e.imageFilename)
+	args = append(args, e.spec.Command...)
+
 	path, err := exec.LookPath(args[0])
 	if err != nil {
 		return err