17813: Singularity image caching wip

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 412f1bbfbfa95027eb5c043c5e1fcf07449139b0..59ef73e65ad45eeef7d15629a4334db22f641804 100644 (file)
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -1559,6 +1559,8 @@ func (runner *ContainerRunner) fetchContainerRecord() error {
        }
        runner.SecretMounts = sm.SecretMounts
 
+       runner.executor.SetArvadoClient(runner.containerClient, runner.Container)
+
        return nil
 }
 

diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go
index 861f8c8c1913f07bab8d7ea722dfa3c643678059..05e540da8083696b8f0f4c08d5778a826f733518 100644 (file)
--- a/lib/crunchrun/docker.go
+++ b/lib/crunchrun/docker.go
@@ -11,6 +11,7 @@ import (
        "strings"
        "time"
 
+       "git.arvados.org/arvados.git/sdk/go/arvados"
        dockertypes "github.com/docker/docker/api/types"
        dockercontainer "github.com/docker/docker/api/types/container"
        dockerclient "github.com/docker/docker/client"
@@ -252,3 +253,6 @@ func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.Writer, reader io.
 func (e *dockerExecutor) Close() {
        e.dockerclient.ContainerRemove(context.TODO(), e.containerID, dockertypes.ContainerRemoveOptions{Force: true})
 }
+
+func (e *dockerExecutor) SetArvadoClient(containerClient *arvados.Client, container arvados.Container) {
+}

diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go
index f4feaa06c21447cc66b2e57a962e2d2c306e6de7..09258ed9db7d161c9bf25d42ec7a127a7baa9f89 100644 (file)
--- a/lib/crunchrun/executor.go
+++ b/lib/crunchrun/executor.go
@@ -6,6 +6,7 @@ package crunchrun
 import (
        "io"
 
+       "git.arvados.org/arvados.git/sdk/go/arvados"
        "golang.org/x/net/context"
 )
 
@@ -60,4 +61,7 @@ type containerExecutor interface {
 
        // Release resources (temp dirs, stopped containers)
        Close()
+
+       // Give the executor access to arvados client & container info
+       SetArvadoClient(containerClient *arvados.Client, container arvados.Container)
 }

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index bcaff3bcc88300e51015f16a94751d20a39d5efe..45926b065fa04d743edb15f2e5ac0107f31bb0cd 100644 (file)
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -5,21 +5,26 @@
 package crunchrun
 
 import (
+       "fmt"
        "io/ioutil"
        "os"
        "os/exec"
        "sort"
+       "strings"
        "syscall"
 
+       "git.arvados.org/arvados.git/sdk/go/arvados"
        "golang.org/x/net/context"
 )
 
 type singularityExecutor struct {
-       logf          func(string, ...interface{})
-       spec          containerSpec
-       tmpdir        string
-       child         *exec.Cmd
-       imageFilename string // "sif" image
+       logf            func(string, ...interface{})
+       spec            containerSpec
+       tmpdir          string
+       child           *exec.Cmd
+       imageFilename   string // "sif" image
+       containerClient *arvados.Client
+       container       arvados.Container
 }
 
 func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExecutor, error) {
@@ -33,13 +38,56 @@ func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExec
        }, nil
 }
 
+func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, create bool) (*arvados.Group, error) {
+       var gp arvados.GroupList
+       err := e.containerClient.RequestAndDecode(&gp,
+               arvados.EndpointGroupList.Method,
+               arvados.EndpointGroupList.Path,
+               nil, arvados.ListOptions{Filters: []arvados.Filter{
+                       arvados.Filter{"owner_uuid", "=", ownerUuid},
+                       arvados.Filter{"name", "=", name},
+                       arvados.Filter{"group_class", "=", "project"},
+               }})
+       if err != nil {
+               return nil, err
+       }
+       if len(gp.Items) > 0 {
+               return &gp.Items[0], nil
+       }
+       if !create {
+               return nil, nil
+       }
+       var rgroup arvados.Group
+       err = e.containerClient.RequestAndDecode(&rgroup,
+               arvados.EndpointGroupCreate.Method,
+               arvados.EndpointGroupCreate.Path,
+               nil, map[string]interface{}{
+                       "group": map[string]string{
+                               "owner_uuid":  ownerUuid,
+                               "name":        name,
+                               "group_class": "project",
+                       },
+               })
+       if err != nil {
+               return nil, err
+       }
+       return &rgroup, nil
+}
+
 func (e *singularityExecutor) ImageLoaded(string) bool {
+       // Check if docker image is cached in keep & if so set imageFilename
+
        return false
 }
 
 // LoadImage will satisfy ContainerExecuter interface transforming
 // containerImage into a sif file for later use.
 func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
+       if e.imageFilename != "" {
+               // was set by ImageLoaded
+               return nil
+       }
+
        e.logf("building singularity image")
        // "singularity build" does not accept a
        // docker-archive://... filename containing a ":" character,
@@ -66,6 +114,36 @@ func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
        if err != nil {
                return err
        }
+
+       // Cache the image to keep
+       cacheGroup, err := e.getOrCreateProject(e.container.RuntimeUserUUID, ".cache", true)
+       if err != nil {
+               e.logf("error getting '.cache' project: %s", err)
+               return nil
+       }
+       imageGroup, err := e.getOrCreateProject(cacheGroup.UUID, "auto-generated singularity images", true)
+       if err != nil {
+               e.logf("error getting 'auto-generated singularity images' project: %s", err)
+               return nil
+       }
+
+       parts := strings.Split(imageTarballPath, "/")
+       imageId := parts[len(parts)-1]
+
+       var imageCollection arvados.Collection
+       err = e.containerClient.RequestAndDecode(&imageCollection,
+               arvados.EndpointCollectionCreate.Method,
+               arvados.EndpointCollectionCreate.Path,
+               nil, map[string]interface{}{
+                       "collection": map[string]string{
+                               "owner_uuid": imageGroup.UUID,
+                               "name": fmt.Sprintf("singularity image for %s", imageId),
+                       }
+               })
+       if err != nil {
+               e.logf("error creating 'auto-generated singularity images' collection: %s", err)
+       }
+
        return nil
 }
 
@@ -153,3 +231,8 @@ func (e *singularityExecutor) Close() {
                e.logf("error removing temp dir: %s", err)
        }
 }
+
+func (e *singularityExecutor) SetArvadoClient(containerClient *arvados.Client, container arvados.Container) {
+       e.containerClient = containerClient
+       e.container = container
+}

diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index b57dc849442f4934f10611acd0248539af3a827e..384bebb5997ee86b1b1be2396498f1554ee32ecc 100644 (file)
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -33,6 +33,9 @@ type Container struct {
        GatewayAddress            string                 `json:"gateway_address"`
        InteractiveSessionStarted bool                   `json:"interactive_session_started"`
        OutputStorageClasses      []string               `json:"output_storage_classes"`
+       RuntimeUserUUID           string                 `json:"runtime_user_uuid"`
+       RuntimeAuthScopes         []string               `json:"runtime_auth_scopes"`
+       RuntimeToken              string                 `json:"runtime_token"`
 }
 
 // ContainerRequest is an arvados#container_request resource.