closes #7453
[arvados.git] / sdk / go / crunchrunner / crunchrunner_test.go
1 package main
2
3 import (
4         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
5         . "gopkg.in/check.v1"
6         "io"
7         "io/ioutil"
8         "log"
9         "os"
10         "syscall"
11         "testing"
12         "time"
13 )
14
15 // Gocheck boilerplate
16 func Test(t *testing.T) {
17         TestingT(t)
18 }
19
20 type TestSuite struct{}
21
22 // Gocheck boilerplate
23 var _ = Suite(&TestSuite{})
24
25 type ArvTestClient struct {
26         c        *C
27         manifest string
28         success  bool
29 }
30
31 func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
32         return nil
33 }
34
35 func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
36         t.c.Check(resourceType, Equals, "job_tasks")
37         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
38                 Output:   t.manifest,
39                 Success:  t.success,
40                 Progress: 1}})
41         return nil
42 }
43
44 func (s *TestSuite) TestSimpleRun(c *C) {
45         tmpdir, _ := ioutil.TempDir("", "")
46         defer func() {
47                 os.RemoveAll(tmpdir)
48         }()
49
50         err := runner(ArvTestClient{c, "", true},
51                 KeepTestClient{},
52                 "zzzz-8i9sb-111111111111111",
53                 "zzzz-ot0gb-111111111111111",
54                 tmpdir,
55                 "",
56                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
57                         Command: []string{"echo", "foo"}}}}},
58                 Task{Sequence: 0})
59         c.Check(err, IsNil)
60 }
61
62 func checkOutput(c *C, tmpdir string) {
63         file, err := os.Open(tmpdir + "/outdir/output.txt")
64         c.Assert(err, IsNil)
65
66         data := make([]byte, 100)
67         var count int
68         err = nil
69         offset := 0
70         for err == nil {
71                 count, err = file.Read(data[offset:])
72                 offset += count
73         }
74         c.Assert(err, Equals, io.EOF)
75         c.Check(string(data[0:offset]), Equals, "foo\n")
76 }
77
78 func (s *TestSuite) TestSimpleRunSubtask(c *C) {
79         tmpdir, _ := ioutil.TempDir("", "")
80         defer func() {
81                 os.RemoveAll(tmpdir)
82         }()
83
84         err := runner(ArvTestClient{c,
85                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
86                 KeepTestClient{},
87                 "zzzz-8i9sb-111111111111111",
88                 "zzzz-ot0gb-111111111111111",
89                 tmpdir,
90                 "",
91                 Job{Script_parameters: Tasks{[]TaskDef{
92                         TaskDef{Command: []string{"echo", "bar"}},
93                         TaskDef{Command: []string{"echo", "foo"}}}}},
94                 Task{Parameters: TaskDef{
95                         Command: []string{"echo", "foo"},
96                         Stdout:  "output.txt"},
97                         Sequence: 1})
98         c.Check(err, IsNil)
99
100         checkOutput(c, tmpdir)
101 }
102
103 func (s *TestSuite) TestRedirect(c *C) {
104         tmpfile, _ := ioutil.TempFile("", "")
105         tmpfile.Write([]byte("foo\n"))
106         tmpfile.Close()
107         defer os.Remove(tmpfile.Name())
108
109         tmpdir, _ := ioutil.TempDir("", "")
110         defer func() {
111                 os.RemoveAll(tmpdir)
112         }()
113
114         err := runner(ArvTestClient{c,
115                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
116                 KeepTestClient{},
117                 "zzzz-8i9sb-111111111111111",
118                 "zzzz-ot0gb-111111111111111",
119                 tmpdir,
120                 "",
121                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
122                         Command: []string{"cat"},
123                         Stdout:  "output.txt",
124                         Stdin:   tmpfile.Name()}}}},
125                 Task{Sequence: 0})
126         c.Check(err, IsNil)
127
128         checkOutput(c, tmpdir)
129 }
130
131 func (s *TestSuite) TestEnv(c *C) {
132         tmpdir, _ := ioutil.TempDir("", "")
133         defer func() {
134                 os.RemoveAll(tmpdir)
135         }()
136
137         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
138                 KeepTestClient{},
139                 "zzzz-8i9sb-111111111111111",
140                 "zzzz-ot0gb-111111111111111",
141                 tmpdir,
142                 "",
143                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
144                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
145                         Stdout:  "output.txt",
146                         Env:     map[string]string{"BAR": "foo"}}}}},
147                 Task{Sequence: 0})
148         c.Check(err, IsNil)
149         checkOutput(c, tmpdir)
150 }
151
152 func (s *TestSuite) TestEnvSubstitute(c *C) {
153         tmpdir, _ := ioutil.TempDir("", "")
154         defer func() {
155                 os.RemoveAll(tmpdir)
156         }()
157
158         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
159                 KeepTestClient{},
160                 "zzzz-8i9sb-111111111111111",
161                 "zzzz-ot0gb-111111111111111",
162                 tmpdir,
163                 "foo\n",
164                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
165                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
166                         Stdout:  "output.txt",
167                         Env:     map[string]string{"BAR": "$(task.keep)"}}}}},
168                 Task{Sequence: 0})
169         c.Check(err, IsNil)
170         checkOutput(c, tmpdir)
171 }
172
173 func (s *TestSuite) TestEnvReplace(c *C) {
174         tmpdir, _ := ioutil.TempDir("", "")
175         defer func() {
176                 os.RemoveAll(tmpdir)
177         }()
178
179         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
180                 KeepTestClient{},
181                 "zzzz-8i9sb-111111111111111",
182                 "zzzz-ot0gb-111111111111111",
183                 tmpdir,
184                 "",
185                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
186                         Command: []string{"/bin/sh", "-c", "echo $PATH"},
187                         Stdout:  "output.txt",
188                         Env:     map[string]string{"PATH": "foo"}}}}},
189                 Task{Sequence: 0})
190         c.Check(err, IsNil)
191         checkOutput(c, tmpdir)
192 }
193
194 type SubtaskTestClient struct {
195         c     *C
196         parms []Task
197         i     int
198 }
199
200 func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
201         t.c.Check(resourceType, Equals, "job_tasks")
202         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": t.parms[t.i]})
203         t.i += 1
204         return nil
205 }
206
207 func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
208         return nil
209 }
210
211 func (s *TestSuite) TestScheduleSubtask(c *C) {
212
213         api := SubtaskTestClient{c, []Task{
214                 Task{Job_uuid: "zzzz-8i9sb-111111111111111",
215                         Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
216                         Sequence:                 1,
217                         Parameters: TaskDef{
218                                 Command: []string{"echo", "bar"}}},
219                 Task{Job_uuid: "zzzz-8i9sb-111111111111111",
220                         Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
221                         Sequence:                 1,
222                         Parameters: TaskDef{
223                                 Command: []string{"echo", "foo"}}}},
224                 0}
225
226         tmpdir, _ := ioutil.TempDir("", "")
227         defer func() {
228                 os.RemoveAll(tmpdir)
229         }()
230
231         err := runner(&api, KeepTestClient{},
232                 "zzzz-8i9sb-111111111111111",
233                 "zzzz-ot0gb-111111111111111",
234                 tmpdir,
235                 "",
236                 Job{Script_parameters: Tasks{[]TaskDef{
237                         TaskDef{Command: []string{"echo", "bar"}},
238                         TaskDef{Command: []string{"echo", "foo"}}}}},
239                 Task{Sequence: 0})
240         c.Check(err, IsNil)
241
242 }
243
244 func (s *TestSuite) TestRunFail(c *C) {
245         tmpdir, _ := ioutil.TempDir("", "")
246         defer func() {
247                 os.RemoveAll(tmpdir)
248         }()
249
250         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
251                 "zzzz-8i9sb-111111111111111",
252                 "zzzz-ot0gb-111111111111111",
253                 tmpdir,
254                 "",
255                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
256                         Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
257                 Task{Sequence: 0})
258         c.Check(err, FitsTypeOf, PermFail{})
259 }
260
261 func (s *TestSuite) TestRunSuccessCode(c *C) {
262         tmpdir, _ := ioutil.TempDir("", "")
263         defer func() {
264                 os.RemoveAll(tmpdir)
265         }()
266
267         err := runner(ArvTestClient{c, "", true}, KeepTestClient{},
268                 "zzzz-8i9sb-111111111111111",
269                 "zzzz-ot0gb-111111111111111",
270                 tmpdir,
271                 "",
272                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
273                         Command:      []string{"/bin/sh", "-c", "exit 1"},
274                         SuccessCodes: []int{0, 1}}}}},
275                 Task{Sequence: 0})
276         c.Check(err, IsNil)
277 }
278
279 func (s *TestSuite) TestRunFailCode(c *C) {
280         tmpdir, _ := ioutil.TempDir("", "")
281         defer func() {
282                 os.RemoveAll(tmpdir)
283         }()
284
285         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
286                 "zzzz-8i9sb-111111111111111",
287                 "zzzz-ot0gb-111111111111111",
288                 tmpdir,
289                 "",
290                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
291                         Command:            []string{"/bin/sh", "-c", "exit 0"},
292                         PermanentFailCodes: []int{0, 1}}}}},
293                 Task{Sequence: 0})
294         c.Check(err, FitsTypeOf, PermFail{})
295 }
296
297 func (s *TestSuite) TestRunTempFailCode(c *C) {
298         tmpdir, _ := ioutil.TempDir("", "")
299         defer func() {
300                 os.RemoveAll(tmpdir)
301         }()
302
303         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
304                 "zzzz-8i9sb-111111111111111",
305                 "zzzz-ot0gb-111111111111111",
306                 tmpdir,
307                 "",
308                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
309                         Command:            []string{"/bin/sh", "-c", "exit 1"},
310                         TemporaryFailCodes: []int{1}}}}},
311                 Task{Sequence: 0})
312         c.Check(err, FitsTypeOf, TempFail{})
313 }
314
315 func (s *TestSuite) TestVwd(c *C) {
316         tmpfile, _ := ioutil.TempFile("", "")
317         tmpfile.Write([]byte("foo\n"))
318         tmpfile.Close()
319         defer os.Remove(tmpfile.Name())
320
321         tmpdir, _ := ioutil.TempDir("", "")
322         defer func() {
323                 os.RemoveAll(tmpdir)
324         }()
325
326         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
327                 KeepTestClient{},
328                 "zzzz-8i9sb-111111111111111",
329                 "zzzz-ot0gb-111111111111111",
330                 tmpdir,
331                 "",
332                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
333                         Command: []string{"ls", "output.txt"},
334                         Vwd: map[string]string{
335                                 "output.txt": tmpfile.Name()}}}}},
336                 Task{Sequence: 0})
337         c.Check(err, IsNil)
338         checkOutput(c, tmpdir)
339 }
340
341 func (s *TestSuite) TestSubstitutionStdin(c *C) {
342         keepmount, _ := ioutil.TempDir("", "")
343         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
344         defer func() {
345                 os.RemoveAll(keepmount)
346         }()
347
348         log.Print("Keepmount is ", keepmount)
349
350         tmpdir, _ := ioutil.TempDir("", "")
351         defer func() {
352                 os.RemoveAll(tmpdir)
353         }()
354
355         log.Print("tmpdir is ", tmpdir)
356
357         err := runner(ArvTestClient{c,
358                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
359                 KeepTestClient{},
360                 "zzzz-8i9sb-111111111111111",
361                 "zzzz-ot0gb-111111111111111",
362                 tmpdir,
363                 keepmount,
364                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
365                         Command: []string{"cat"},
366                         Stdout:  "output.txt",
367                         Stdin:   "$(task.keep)/file1.txt"}}}},
368                 Task{Sequence: 0})
369         c.Check(err, IsNil)
370         checkOutput(c, tmpdir)
371 }
372
373 func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
374         keepmount, _ := ioutil.TempDir("", "")
375         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
376         defer func() {
377                 os.RemoveAll(keepmount)
378         }()
379
380         tmpdir, _ := ioutil.TempDir("", "")
381         defer func() {
382                 os.RemoveAll(tmpdir)
383         }()
384
385         err := runner(ArvTestClient{c,
386                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
387                 KeepTestClient{},
388                 "zzzz-8i9sb-111111111111111",
389                 "zzzz-ot0gb-111111111111111",
390                 tmpdir,
391                 keepmount,
392                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
393                         Command: []string{"cat", "$(task.keep)/file1.txt"},
394                         Stdout:  "output.txt"}}}},
395                 Task{Sequence: 0})
396         c.Check(err, IsNil)
397
398         checkOutput(c, tmpdir)
399 }
400
401 func (s *TestSuite) TestSignal(c *C) {
402         tmpdir, _ := ioutil.TempDir("", "")
403         defer func() {
404                 os.RemoveAll(tmpdir)
405         }()
406
407         go func() {
408                 time.Sleep(1 * time.Second)
409                 self, _ := os.FindProcess(os.Getpid())
410                 self.Signal(syscall.SIGINT)
411         }()
412
413         err := runner(ArvTestClient{c,
414                 "", false},
415                 KeepTestClient{},
416                 "zzzz-8i9sb-111111111111111",
417                 "zzzz-ot0gb-111111111111111",
418                 tmpdir,
419                 "",
420                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
421                         Command: []string{"sleep", "4"}}}}},
422                 Task{Sequence: 0})
423         c.Check(err, FitsTypeOf, PermFail{})
424
425 }
426
427 func (s *TestSuite) TestQuoting(c *C) {
428         tmpdir, _ := ioutil.TempDir("", "")
429         defer func() {
430                 os.RemoveAll(tmpdir)
431         }()
432
433         err := runner(ArvTestClient{c,
434                 "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
435                 KeepTestClient{},
436                 "zzzz-8i9sb-111111111111111",
437                 "zzzz-ot0gb-111111111111111",
438                 tmpdir,
439                 "",
440                 Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
441                         Command: []string{"echo", "foo"},
442                         Stdout:  "s ub:dir/:e vi\nl"}}}},
443                 Task{Sequence: 0})
444         c.Check(err, IsNil)
445 }