2411: Add copyright notices to everything.

[arvados.git] / sdk / go / crunchrunner / crunchrunner_test.go

diff --git a/sdk/go/crunchrunner/crunchrunner_test.go b/sdk/go/crunchrunner/crunchrunner_test.go
index e67c9ee0c13901fda2ef78cec255be7f769fe0e5..f2827c6a3577c2793a24e7921ae33869533b6bc0 100644 (file)
--- a/sdk/go/crunchrunner/crunchrunner_test.go
+++ b/sdk/go/crunchrunner/crunchrunner_test.go
@@ -1,11 +1,20 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package main
 
 import (
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       . "gopkg.in/check.v1"
+       "io"
        "io/ioutil"
+       "log"
        "os"
+       "syscall"
        "testing"
+       "time"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+       . "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -28,24 +37,12 @@ func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict
        return nil
 }
 
-func (t ArvTestClient) Delete(resource string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
-       return nil
-}
-
 func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
        t.c.Check(resourceType, Equals, "job_tasks")
        t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
-               output:   t.manifest,
-               success:  t.success,
-               progress: 1}})
-       return nil
-}
-
-func (t ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
-       return nil
-}
-
-func (t ArvTestClient) List(resource string, parameters arvadosclient.Dict, output interface{}) (err error) {
+               Output:   t.manifest,
+               Success:  t.success,
+               Progress: 1}})
        return nil
 }
 
@@ -61,22 +58,26 @@ func (s *TestSuite) TestSimpleRun(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands: []string{"echo", "foo"}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"echo", "foo"}}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
-
 }
 
 func checkOutput(c *C, tmpdir string) {
-       file, err := os.Open(tmpdir + "/zzzz-ot0gb-111111111111111/output.txt")
+       file, err := os.Open(tmpdir + "/outdir/output.txt")
        c.Assert(err, IsNil)
 
        data := make([]byte, 100)
        var count int
-       count, err = file.Read(data)
-       c.Assert(err, IsNil)
-       c.Check(string(data[0:count]), Equals, "foo\n")
+       err = nil
+       offset := 0
+       for err == nil {
+               count, err = file.Read(data[offset:])
+               offset += count
+       }
+       c.Assert(err, Equals, io.EOF)
+       c.Check(string(data[0:offset]), Equals, "foo\n")
 }
 
 func (s *TestSuite) TestSimpleRunSubtask(c *C) {
@@ -92,13 +93,13 @@ func (s *TestSuite) TestSimpleRunSubtask(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{
-                       TaskDef{commands: []string{"echo", "bar"}},
-                       TaskDef{commands: []string{"echo", "foo"}}}}},
-               Task{parameters: TaskDef{
-                       commands: []string{"echo", "foo"},
-                       stdout:   "output.txt"},
-                       sequence: 1})
+               Job{ScriptParameters: Tasks{[]TaskDef{
+                       {Command: []string{"echo", "bar"}},
+                       {Command: []string{"echo", "foo"}}}}},
+               Task{Parameters: TaskDef{
+                       Command: []string{"echo", "foo"},
+                       Stdout:  "output.txt"},
+                       Sequence: 1})
        c.Check(err, IsNil)
 
        checkOutput(c, tmpdir)
@@ -122,11 +123,11 @@ func (s *TestSuite) TestRedirect(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands: []string{"cat"},
-                       stdout:   "output.txt",
-                       stdin:    tmpfile.Name()}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"cat"},
+                       Stdout:  "output.txt",
+                       Stdin:   tmpfile.Name()}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
 
        checkOutput(c, tmpdir)
@@ -144,13 +145,54 @@ func (s *TestSuite) TestEnv(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands: []string{"/bin/sh", "-c", "echo $BAR"},
-                       stdout:   "output.txt",
-                       env:      map[string]string{"BAR": "foo"}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"/bin/sh", "-c", "echo $BAR"},
+                       Stdout:  "output.txt",
+                       Env:     map[string]string{"BAR": "foo"}}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
+       checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestEnvSubstitute(c *C) {
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               "foo\n",
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"/bin/sh", "-c", "echo $BAR"},
+                       Stdout:  "output.txt",
+                       Env:     map[string]string{"BAR": "$(task.keep)"}}}}},
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
+       checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestEnvReplace(c *C) {
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
 
+       err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               "",
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"/bin/sh", "-c", "echo $PATH"},
+                       Stdout:  "output.txt",
+                       Env:     map[string]string{"PATH": "foo"}}}}},
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
        checkOutput(c, tmpdir)
 }
 
@@ -167,35 +209,23 @@ func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient
        return nil
 }
 
-func (t SubtaskTestClient) Delete(resource string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
-       return nil
-}
-
 func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
        return nil
 }
 
-func (t SubtaskTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
-       return nil
-}
-
-func (t SubtaskTestClient) List(resource string, parameters arvadosclient.Dict, output interface{}) (err error) {
-       return nil
-}
-
 func (s *TestSuite) TestScheduleSubtask(c *C) {
 
        api := SubtaskTestClient{c, []Task{
-               Task{job_uuid: "zzzz-8i9sb-111111111111111",
-                       created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
-                       sequence:                 1,
-                       parameters: TaskDef{
-                               commands: []string{"echo", "bar"}}},
-               Task{job_uuid: "zzzz-8i9sb-111111111111111",
-                       created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
-                       sequence:                 1,
-                       parameters: TaskDef{
-                               commands: []string{"echo", "foo"}}}},
+               {JobUUID: "zzzz-8i9sb-111111111111111",
+                       CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
+                       Sequence:             1,
+                       Parameters: TaskDef{
+                               Command: []string{"echo", "bar"}}},
+               {JobUUID: "zzzz-8i9sb-111111111111111",
+                       CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
+                       Sequence:             1,
+                       Parameters: TaskDef{
+                               Command: []string{"echo", "foo"}}}},
                0}
 
        tmpdir, _ := ioutil.TempDir("", "")
@@ -208,10 +238,10 @@ func (s *TestSuite) TestScheduleSubtask(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{
-                       TaskDef{commands: []string{"echo", "bar"}},
-                       TaskDef{commands: []string{"echo", "foo"}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{
+                       {Command: []string{"echo", "bar"}},
+                       {Command: []string{"echo", "foo"}}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
 
 }
@@ -227,9 +257,9 @@ func (s *TestSuite) TestRunFail(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands: []string{"/bin/sh", "-c", "exit 1"}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
+               Task{Sequence: 0})
        c.Check(err, FitsTypeOf, PermFail{})
 }
 
@@ -244,10 +274,10 @@ func (s *TestSuite) TestRunSuccessCode(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands:     []string{"/bin/sh", "-c", "exit 1"},
-                       successCodes: []int{0, 1}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command:      []string{"/bin/sh", "-c", "exit 1"},
+                       SuccessCodes: []int{0, 1}}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
 }
 
@@ -262,10 +292,10 @@ func (s *TestSuite) TestRunFailCode(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands:           []string{"/bin/sh", "-c", "exit 0"},
-                       permanentFailCodes: []int{0, 1}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command:            []string{"/bin/sh", "-c", "exit 0"},
+                       PermanentFailCodes: []int{0, 1}}}}},
+               Task{Sequence: 0})
        c.Check(err, FitsTypeOf, PermFail{})
 }
 
@@ -280,10 +310,10 @@ func (s *TestSuite) TestRunTempFailCode(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands:           []string{"/bin/sh", "-c", "exit 1"},
-                       temporaryFailCodes: []int{1}}}}},
-               Task{sequence: 0})
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command:            []string{"/bin/sh", "-c", "exit 1"},
+                       TemporaryFailCodes: []int{1}}}}},
+               Task{Sequence: 0})
        c.Check(err, FitsTypeOf, TempFail{})
 }
 
@@ -304,11 +334,145 @@ func (s *TestSuite) TestVwd(c *C) {
                "zzzz-ot0gb-111111111111111",
                tmpdir,
                "",
-               Job{script_parameters: Tasks{[]TaskDef{TaskDef{
-                       commands: []string{"ls", "output.txt"},
-                       vwd: map[string]string{
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"ls", "output.txt"},
+                       Vwd: map[string]string{
                                "output.txt": tmpfile.Name()}}}}},
-               Task{sequence: 0})
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
+       checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestSubstitutionStdin(c *C) {
+       keepmount, _ := ioutil.TempDir("", "")
+       ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
+       defer func() {
+               os.RemoveAll(keepmount)
+       }()
+
+       log.Print("Keepmount is ", keepmount)
+
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       log.Print("tmpdir is ", tmpdir)
+
+       err := runner(ArvTestClient{c,
+               ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               keepmount,
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"cat"},
+                       Stdout:  "output.txt",
+                       Stdin:   "$(task.keep)/file1.txt"}}}},
+               Task{Sequence: 0})
        c.Check(err, IsNil)
        checkOutput(c, tmpdir)
 }
+
+func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
+       keepmount, _ := ioutil.TempDir("", "")
+       ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
+       defer func() {
+               os.RemoveAll(keepmount)
+       }()
+
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       err := runner(ArvTestClient{c,
+               ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               keepmount,
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"cat", "$(task.keep)/file1.txt"},
+                       Stdout:  "output.txt"}}}},
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
+
+       checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestSignal(c *C) {
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       go func() {
+               time.Sleep(1 * time.Second)
+               self, _ := os.FindProcess(os.Getpid())
+               self.Signal(syscall.SIGINT)
+       }()
+
+       err := runner(ArvTestClient{c,
+               "", false},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               "",
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"sleep", "4"}}}}},
+               Task{Sequence: 0})
+       c.Check(err, FitsTypeOf, PermFail{})
+
+}
+
+func (s *TestSuite) TestQuoting(c *C) {
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       err := runner(ArvTestClient{c,
+               "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               "",
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command: []string{"echo", "foo"},
+                       Stdout:  "s ub:dir/:e vi\nl"}}}},
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
+}
+
+func (s *TestSuite) TestKeepTmp(c *C) {
+       tmpdir, _ := ioutil.TempDir("", "")
+       defer func() {
+               os.RemoveAll(tmpdir)
+       }()
+
+       os.Setenv("TASK_KEEPMOUNT_TMP", tmpdir)
+       defer os.Setenv("TASK_KEEPMOUNT_TMP", "")
+
+       fn, err := os.Create(tmpdir + "/.arvados#collection")
+       fn.Write([]byte("{\"manifest_text\":\". unparsed 0:3:foo\\n\",\"uuid\":null}"))
+       defer fn.Close()
+
+       err = runner(ArvTestClient{c,
+               ". unparsed 0:3:foo\n", true},
+               KeepTestClient{},
+               "zzzz-8i9sb-111111111111111",
+               "zzzz-ot0gb-111111111111111",
+               tmpdir,
+               "",
+               Job{ScriptParameters: Tasks{[]TaskDef{{
+                       Command:       []string{"echo", "foo"},
+                       KeepTmpOutput: true}}}},
+               Task{Sequence: 0})
+       c.Check(err, IsNil)
+
+}