X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a206b8127df92039645d5ad3f9e144380cc55613..3c34e713aa343d56c7cea00a9c998b06dbf411d6:/sdk/go/crunchrunner/crunchrunner.go

diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index 040d7c20be..36343d3df3 100644
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -1,13 +1,14 @@
 package main
 
 import (
-	"crypto/x509"
+	"encoding/json"
 	"fmt"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -25,6 +26,7 @@ type TaskDef struct {
 	SuccessCodes       []int             `json:"task.successCodes"`
 	PermanentFailCodes []int             `json:"task.permanentFailCodes"`
 	TemporaryFailCodes []int             `json:"task.temporaryFailCodes"`
+	KeepTmpOutput      bool              `json:"task.keepTmpOutput"`
 }
 
 type Tasks struct {
@@ -50,17 +52,21 @@ type IArvadosClient interface {
 	Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
 }
 
-func setupDirectories(crunchtmpdir, taskUuid string) (tmpdir, outdir string, err error) {
+func setupDirectories(crunchtmpdir, taskUuid string, keepTmp bool) (tmpdir, outdir string, err error) {
 	tmpdir = crunchtmpdir + "/tmpdir"
 	err = os.Mkdir(tmpdir, 0700)
 	if err != nil {
 		return "", "", err
 	}
 
-	outdir = crunchtmpdir + "/outdir"
-	err = os.Mkdir(outdir, 0700)
-	if err != nil {
-		return "", "", err
+	if keepTmp {
+		outdir = os.Getenv("TASK_KEEPMOUNT_TMP")
+	} else {
+		outdir = crunchtmpdir + "/outdir"
+		err = os.Mkdir(outdir, 0700)
+		if err != nil {
+			return "", "", err
+		}
 	}
 
 	return tmpdir, outdir, nil
@@ -81,6 +87,23 @@ func checkOutputFilename(outdir, fn string) error {
 	return nil
 }
 
+func copyFile(dst, src string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
 func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[string]string) (stdin, stdout, stderr string, err error) {
 	if taskp.Vwd != nil {
 		for k, v := range taskp.Vwd {
@@ -89,7 +112,14 @@ func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[
 			if err != nil {
 				return "", "", "", err
 			}
-			os.Symlink(v, outdir+"/"+k)
+			if taskp.KeepTmpOutput {
+				err = copyFile(v, outdir+"/"+k)
+			} else {
+				err = os.Symlink(v, outdir+"/"+k)
+			}
+			if err != nil {
+				return "", "", "", err
+			}
 		}
 	}
 
@@ -180,6 +210,22 @@ func substitute(inp string, subst map[string]string) string {
 	return inp
 }
 
+func getKeepTmp(outdir string) (manifest string, err error) {
+	fn, err := os.Open(outdir + "/" + ".arvados#collection")
+	if err != nil {
+		return "", err
+	}
+	defer fn.Close()
+
+	buf, err := ioutil.ReadAll(fn)
+	if err != nil {
+		return "", err
+	}
+	collection := arvados.Collection{}
+	err = json.Unmarshal(buf, &collection)
+	return collection.ManifestText, err
+}
+
 func runner(api IArvadosClient,
 	kc IKeepClient,
 	jobUuid, taskUuid, crunchtmpdir, keepmount string,
@@ -208,17 +254,17 @@ func runner(api IArvadosClient,
 			}
 			err = api.Update("job_tasks", taskUuid,
 				map[string]interface{}{
-					"job_task": Task{
-						Output:   "",
-						Success:  true,
-						Progress: 1.0}},
+					"job_task": map[string]interface{}{
+						"output":   "",
+						"success":  true,
+						"progress": 1.0}},
 				nil)
 			return nil
 		}
 	}
 
 	var tmpdir, outdir string
-	tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid)
+	tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid, taskp.KeepTmpOutput)
 	if err != nil {
 		return TempFail{err}
 	}
@@ -313,7 +359,12 @@ func runner(api IArvadosClient,
 	}
 
 	// Upload output directory
-	manifest, err := WriteTree(kc, outdir)
+	var manifest string
+	if taskp.KeepTmpOutput {
+		manifest, err = getKeepTmp(outdir)
+	} else {
+		manifest, err = WriteTree(kc, outdir)
+	}
 	if err != nil {
 		return TempFail{err}
 	}
@@ -343,24 +394,6 @@ func main() {
 		log.Fatal(err)
 	}
 
-	// Container may not have certificates installed, so need to look for
-	// /etc/arvados/ca-certificates.crt in addition to normal system certs.
-	var certFiles = []string{
-		"/etc/ssl/certs/ca-certificates.crt", // Debian
-		"/etc/pki/tls/certs/ca-bundle.crt",   // Red Hat
-		"/etc/arvados/ca-certificates.crt",
-	}
-
-	certs := x509.NewCertPool()
-	for _, file := range certFiles {
-		data, err := ioutil.ReadFile(file)
-		if err == nil {
-			log.Printf("Using TLS certificates at %v", file)
-			certs.AppendCertsFromPEM(data)
-		}
-	}
-	api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
-
 	jobUuid := os.Getenv("JOB_UUID")
 	taskUuid := os.Getenv("TASK_UUID")
 	tmpdir := os.Getenv("TASK_WORK")
@@ -379,7 +412,7 @@ func main() {
 	}
 
 	var kc IKeepClient
-	kc, err = keepclient.MakeKeepClient(&api)
+	kc, err = keepclient.MakeKeepClient(api)
 	if err != nil {
 		log.Fatal(err)
 	}