Merge branch '11070-ws-listener-problem'

[arvados.git] / sdk / go / crunchrunner / crunchrunner.go

diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index 5c3d65c56144bab68e322b1040c0fb846e30ddaa..5d7e10be4beb34fef1892b2d2d7c150fd9906176 100644 (file)
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -1,13 +1,14 @@
 package main
 
 import (
-       "crypto/x509"
+       "encoding/json"
        "fmt"
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "io"
        "io/ioutil"
        "log"
-       "net/http"
        "os"
        "os/exec"
        "os/signal"
@@ -25,6 +26,7 @@ type TaskDef struct {
        SuccessCodes       []int             `json:"task.successCodes"`
        PermanentFailCodes []int             `json:"task.permanentFailCodes"`
        TemporaryFailCodes []int             `json:"task.temporaryFailCodes"`
+       KeepTmpOutput      bool              `json:"task.keepTmpOutput"`
 }
 
 type Tasks struct {
@@ -50,17 +52,21 @@ type IArvadosClient interface {
        Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
 }
 
-func setupDirectories(crunchtmpdir, taskUuid string) (tmpdir, outdir string, err error) {
+func setupDirectories(crunchtmpdir, taskUuid string, keepTmp bool) (tmpdir, outdir string, err error) {
        tmpdir = crunchtmpdir + "/tmpdir"
        err = os.Mkdir(tmpdir, 0700)
        if err != nil {
                return "", "", err
        }
 
-       outdir = crunchtmpdir + "/outdir"
-       err = os.Mkdir(outdir, 0700)
-       if err != nil {
-               return "", "", err
+       if keepTmp {
+               outdir = os.Getenv("TASK_KEEPMOUNT_TMP")
+       } else {
+               outdir = crunchtmpdir + "/outdir"
+               err = os.Mkdir(outdir, 0700)
+               if err != nil {
+                       return "", "", err
+               }
        }
 
        return tmpdir, outdir, nil
@@ -81,6 +87,23 @@ func checkOutputFilename(outdir, fn string) error {
        return nil
 }
 
+func copyFile(dst, src string) error {
+       in, err := os.Open(src)
+       if err != nil {
+               return err
+       }
+       defer in.Close()
+
+       out, err := os.Create(dst)
+       if err != nil {
+               return err
+       }
+       defer out.Close()
+
+       _, err = io.Copy(out, in)
+       return err
+}
+
 func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[string]string) (stdin, stdout, stderr string, err error) {
        if taskp.Vwd != nil {
                for k, v := range taskp.Vwd {
@@ -89,7 +112,14 @@ func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[
                        if err != nil {
                                return "", "", "", err
                        }
-                       os.Symlink(v, outdir+"/"+k)
+                       if taskp.KeepTmpOutput {
+                               err = copyFile(v, outdir+"/"+k)
+                       } else {
+                               err = os.Symlink(v, outdir+"/"+k)
+                       }
+                       if err != nil {
+                               return "", "", "", err
+                       }
                }
        }
 
@@ -180,6 +210,22 @@ func substitute(inp string, subst map[string]string) string {
        return inp
 }
 
+func getKeepTmp(outdir string) (manifest string, err error) {
+       fn, err := os.Open(outdir + "/" + ".arvados#collection")
+       if err != nil {
+               return "", err
+       }
+       defer fn.Close()
+
+       buf, err := ioutil.ReadAll(fn)
+       if err != nil {
+               return "", err
+       }
+       collection := arvados.Collection{}
+       err = json.Unmarshal(buf, &collection)
+       return collection.ManifestText, err
+}
+
 func runner(api IArvadosClient,
        kc IKeepClient,
        jobUuid, taskUuid, crunchtmpdir, keepmount string,
@@ -218,7 +264,7 @@ func runner(api IArvadosClient,
        }
 
        var tmpdir, outdir string
-       tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid)
+       tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid, taskp.KeepTmpOutput)
        if err != nil {
                return TempFail{err}
        }
@@ -313,7 +359,12 @@ func runner(api IArvadosClient,
        }
 
        // Upload output directory
-       manifest, err := WriteTree(kc, outdir)
+       var manifest string
+       if taskp.KeepTmpOutput {
+               manifest, err = getKeepTmp(outdir)
+       } else {
+               manifest, err = WriteTree(kc, outdir)
+       }
        if err != nil {
                return TempFail{err}
        }
@@ -343,24 +394,6 @@ func main() {
                log.Fatal(err)
        }
 
-       // Container may not have certificates installed, so need to look for
-       // /etc/arvados/ca-certificates.crt in addition to normal system certs.
-       var certFiles = []string{
-               "/etc/ssl/certs/ca-certificates.crt", // Debian
-               "/etc/pki/tls/certs/ca-bundle.crt",   // Red Hat
-               "/etc/arvados/ca-certificates.crt",
-       }
-
-       certs := x509.NewCertPool()
-       for _, file := range certFiles {
-               data, err := ioutil.ReadFile(file)
-               if err == nil {
-                       log.Printf("Using TLS certificates at %v", file)
-                       certs.AppendCertsFromPEM(data)
-               }
-       }
-       api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
-
        jobUuid := os.Getenv("JOB_UUID")
        taskUuid := os.Getenv("TASK_UUID")
        tmpdir := os.Getenv("TASK_WORK")