sdk/go/crunchrunner/crunchrunner_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package main
   6
   7 import (
   8         "io"
   9         "io/ioutil"
  10         "log"
  11         "os"
  12         "syscall"
  13         "testing"
  14         "time"
  15
  16         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 type TestSuite struct{}
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&TestSuite{})
  29
  30 type ArvTestClient struct {
  31         c        *C
  32         manifest string
  33         success  bool
  34 }
  35
  36 func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
  37         return nil
  38 }
  39
  40 func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
  41         t.c.Check(resourceType, Equals, "job_tasks")
  42         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
  43                 Output:   t.manifest,
  44                 Success:  t.success,
  45                 Progress: 1}})
  46         return nil
  47 }
  48
  49 func (s *TestSuite) TestSimpleRun(c *C) {
  50         tmpdir, _ := ioutil.TempDir("", "")
  51         defer func() {
  52                 os.RemoveAll(tmpdir)
  53         }()
  54
  55         err := runner(ArvTestClient{c, "", true},
  56                 KeepTestClient{},
  57                 "zzzz-8i9sb-111111111111111",
  58                 "zzzz-ot0gb-111111111111111",
  59                 tmpdir,
  60                 "",
  61                 Job{ScriptParameters: Tasks{[]TaskDef{{
  62                         Command: []string{"echo", "foo"}}}}},
  63                 Task{Sequence: 0})
  64         c.Check(err, IsNil)
  65 }
  66
  67 func checkOutput(c *C, tmpdir string) {
  68         file, err := os.Open(tmpdir + "/outdir/output.txt")
  69         c.Assert(err, IsNil)
  70
  71         data := make([]byte, 100)
  72         var count int
  73         err = nil
  74         offset := 0
  75         for err == nil {
  76                 count, err = file.Read(data[offset:])
  77                 offset += count
  78         }
  79         c.Assert(err, Equals, io.EOF)
  80         c.Check(string(data[0:offset]), Equals, "foo\n")
  81 }
  82
  83 func (s *TestSuite) TestSimpleRunSubtask(c *C) {
  84         tmpdir, _ := ioutil.TempDir("", "")
  85         defer func() {
  86                 os.RemoveAll(tmpdir)
  87         }()
  88
  89         err := runner(ArvTestClient{c,
  90                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
  91                 KeepTestClient{},
  92                 "zzzz-8i9sb-111111111111111",
  93                 "zzzz-ot0gb-111111111111111",
  94                 tmpdir,
  95                 "",
  96                 Job{ScriptParameters: Tasks{[]TaskDef{
  97                         {Command: []string{"echo", "bar"}},
  98                         {Command: []string{"echo", "foo"}}}}},
  99                 Task{Parameters: TaskDef{
 100                         Command: []string{"echo", "foo"},
 101                         Stdout:  "output.txt"},
 102                         Sequence: 1})
 103         c.Check(err, IsNil)
 104
 105         checkOutput(c, tmpdir)
 106 }
 107
 108 func (s *TestSuite) TestRedirect(c *C) {
 109         tmpfile, _ := ioutil.TempFile("", "")
 110         tmpfile.Write([]byte("foo\n"))
 111         tmpfile.Close()
 112         defer os.Remove(tmpfile.Name())
 113
 114         tmpdir, _ := ioutil.TempDir("", "")
 115         defer func() {
 116                 os.RemoveAll(tmpdir)
 117         }()
 118
 119         err := runner(ArvTestClient{c,
 120                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 121                 KeepTestClient{},
 122                 "zzzz-8i9sb-111111111111111",
 123                 "zzzz-ot0gb-111111111111111",
 124                 tmpdir,
 125                 "",
 126                 Job{ScriptParameters: Tasks{[]TaskDef{{
 127                         Command: []string{"cat"},
 128                         Stdout:  "output.txt",
 129                         Stdin:   tmpfile.Name()}}}},
 130                 Task{Sequence: 0})
 131         c.Check(err, IsNil)
 132
 133         checkOutput(c, tmpdir)
 134 }
 135
 136 func (s *TestSuite) TestEnv(c *C) {
 137         tmpdir, _ := ioutil.TempDir("", "")
 138         defer func() {
 139                 os.RemoveAll(tmpdir)
 140         }()
 141
 142         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 143                 KeepTestClient{},
 144                 "zzzz-8i9sb-111111111111111",
 145                 "zzzz-ot0gb-111111111111111",
 146                 tmpdir,
 147                 "",
 148                 Job{ScriptParameters: Tasks{[]TaskDef{{
 149                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 150                         Stdout:  "output.txt",
 151                         Env:     map[string]string{"BAR": "foo"}}}}},
 152                 Task{Sequence: 0})
 153         c.Check(err, IsNil)
 154         checkOutput(c, tmpdir)
 155 }
 156
 157 func (s *TestSuite) TestEnvSubstitute(c *C) {
 158         tmpdir, _ := ioutil.TempDir("", "")
 159         defer func() {
 160                 os.RemoveAll(tmpdir)
 161         }()
 162
 163         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 164                 KeepTestClient{},
 165                 "zzzz-8i9sb-111111111111111",
 166                 "zzzz-ot0gb-111111111111111",
 167                 tmpdir,
 168                 "foo\n",
 169                 Job{ScriptParameters: Tasks{[]TaskDef{{
 170                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 171                         Stdout:  "output.txt",
 172                         Env:     map[string]string{"BAR": "$(task.keep)"}}}}},
 173                 Task{Sequence: 0})
 174         c.Check(err, IsNil)
 175         checkOutput(c, tmpdir)
 176 }
 177
 178 func (s *TestSuite) TestEnvReplace(c *C) {
 179         tmpdir, _ := ioutil.TempDir("", "")
 180         defer func() {
 181                 os.RemoveAll(tmpdir)
 182         }()
 183
 184         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 185                 KeepTestClient{},
 186                 "zzzz-8i9sb-111111111111111",
 187                 "zzzz-ot0gb-111111111111111",
 188                 tmpdir,
 189                 "",
 190                 Job{ScriptParameters: Tasks{[]TaskDef{{
 191                         Command: []string{"/bin/sh", "-c", "echo $PATH"},
 192                         Stdout:  "output.txt",
 193                         Env:     map[string]string{"PATH": "foo"}}}}},
 194                 Task{Sequence: 0})
 195         c.Check(err, IsNil)
 196         checkOutput(c, tmpdir)
 197 }
 198
 199 type SubtaskTestClient struct {
 200         c     *C
 201         parms []Task
 202         i     int
 203 }
 204
 205 func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
 206         t.c.Check(resourceType, Equals, "job_tasks")
 207         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": t.parms[t.i]})
 208         t.i += 1
 209         return nil
 210 }
 211
 212 func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 213         return nil
 214 }
 215
 216 func (s *TestSuite) TestScheduleSubtask(c *C) {
 217
 218         api := SubtaskTestClient{c, []Task{
 219                 {JobUUID: "zzzz-8i9sb-111111111111111",
 220                         CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
 221                         Sequence:             1,
 222                         Parameters: TaskDef{
 223                                 Command: []string{"echo", "bar"}}},
 224                 {JobUUID: "zzzz-8i9sb-111111111111111",
 225                         CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
 226                         Sequence:             1,
 227                         Parameters: TaskDef{
 228                                 Command: []string{"echo", "foo"}}}},
 229                 0}
 230
 231         tmpdir, _ := ioutil.TempDir("", "")
 232         defer func() {
 233                 os.RemoveAll(tmpdir)
 234         }()
 235
 236         err := runner(&api, KeepTestClient{},
 237                 "zzzz-8i9sb-111111111111111",
 238                 "zzzz-ot0gb-111111111111111",
 239                 tmpdir,
 240                 "",
 241                 Job{ScriptParameters: Tasks{[]TaskDef{
 242                         {Command: []string{"echo", "bar"}},
 243                         {Command: []string{"echo", "foo"}}}}},
 244                 Task{Sequence: 0})
 245         c.Check(err, IsNil)
 246
 247 }
 248
 249 func (s *TestSuite) TestRunFail(c *C) {
 250         tmpdir, _ := ioutil.TempDir("", "")
 251         defer func() {
 252                 os.RemoveAll(tmpdir)
 253         }()
 254
 255         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 256                 "zzzz-8i9sb-111111111111111",
 257                 "zzzz-ot0gb-111111111111111",
 258                 tmpdir,
 259                 "",
 260                 Job{ScriptParameters: Tasks{[]TaskDef{{
 261                         Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
 262                 Task{Sequence: 0})
 263         c.Check(err, FitsTypeOf, PermFail{})
 264 }
 265
 266 func (s *TestSuite) TestRunSuccessCode(c *C) {
 267         tmpdir, _ := ioutil.TempDir("", "")
 268         defer func() {
 269                 os.RemoveAll(tmpdir)
 270         }()
 271
 272         err := runner(ArvTestClient{c, "", true}, KeepTestClient{},
 273                 "zzzz-8i9sb-111111111111111",
 274                 "zzzz-ot0gb-111111111111111",
 275                 tmpdir,
 276                 "",
 277                 Job{ScriptParameters: Tasks{[]TaskDef{{
 278                         Command:      []string{"/bin/sh", "-c", "exit 1"},
 279                         SuccessCodes: []int{0, 1}}}}},
 280                 Task{Sequence: 0})
 281         c.Check(err, IsNil)
 282 }
 283
 284 func (s *TestSuite) TestRunFailCode(c *C) {
 285         tmpdir, _ := ioutil.TempDir("", "")
 286         defer func() {
 287                 os.RemoveAll(tmpdir)
 288         }()
 289
 290         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 291                 "zzzz-8i9sb-111111111111111",
 292                 "zzzz-ot0gb-111111111111111",
 293                 tmpdir,
 294                 "",
 295                 Job{ScriptParameters: Tasks{[]TaskDef{{
 296                         Command:            []string{"/bin/sh", "-c", "exit 0"},
 297                         PermanentFailCodes: []int{0, 1}}}}},
 298                 Task{Sequence: 0})
 299         c.Check(err, FitsTypeOf, PermFail{})
 300 }
 301
 302 func (s *TestSuite) TestRunTempFailCode(c *C) {
 303         tmpdir, _ := ioutil.TempDir("", "")
 304         defer func() {
 305                 os.RemoveAll(tmpdir)
 306         }()
 307
 308         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 309                 "zzzz-8i9sb-111111111111111",
 310                 "zzzz-ot0gb-111111111111111",
 311                 tmpdir,
 312                 "",
 313                 Job{ScriptParameters: Tasks{[]TaskDef{{
 314                         Command:            []string{"/bin/sh", "-c", "exit 1"},
 315                         TemporaryFailCodes: []int{1}}}}},
 316                 Task{Sequence: 0})
 317         c.Check(err, FitsTypeOf, TempFail{})
 318 }
 319
 320 func (s *TestSuite) TestVwd(c *C) {
 321         tmpfile, _ := ioutil.TempFile("", "")
 322         tmpfile.Write([]byte("foo\n"))
 323         tmpfile.Close()
 324         defer os.Remove(tmpfile.Name())
 325
 326         tmpdir, _ := ioutil.TempDir("", "")
 327         defer func() {
 328                 os.RemoveAll(tmpdir)
 329         }()
 330
 331         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 332                 KeepTestClient{},
 333                 "zzzz-8i9sb-111111111111111",
 334                 "zzzz-ot0gb-111111111111111",
 335                 tmpdir,
 336                 "",
 337                 Job{ScriptParameters: Tasks{[]TaskDef{{
 338                         Command: []string{"ls", "output.txt"},
 339                         Vwd: map[string]string{
 340                                 "output.txt": tmpfile.Name()}}}}},
 341                 Task{Sequence: 0})
 342         c.Check(err, IsNil)
 343         checkOutput(c, tmpdir)
 344 }
 345
 346 func (s *TestSuite) TestSubstitutionStdin(c *C) {
 347         keepmount, _ := ioutil.TempDir("", "")
 348         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 349         defer func() {
 350                 os.RemoveAll(keepmount)
 351         }()
 352
 353         log.Print("Keepmount is ", keepmount)
 354
 355         tmpdir, _ := ioutil.TempDir("", "")
 356         defer func() {
 357                 os.RemoveAll(tmpdir)
 358         }()
 359
 360         log.Print("tmpdir is ", tmpdir)
 361
 362         err := runner(ArvTestClient{c,
 363                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 364                 KeepTestClient{},
 365                 "zzzz-8i9sb-111111111111111",
 366                 "zzzz-ot0gb-111111111111111",
 367                 tmpdir,
 368                 keepmount,
 369                 Job{ScriptParameters: Tasks{[]TaskDef{{
 370                         Command: []string{"cat"},
 371                         Stdout:  "output.txt",
 372                         Stdin:   "$(task.keep)/file1.txt"}}}},
 373                 Task{Sequence: 0})
 374         c.Check(err, IsNil)
 375         checkOutput(c, tmpdir)
 376 }
 377
 378 func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
 379         keepmount, _ := ioutil.TempDir("", "")
 380         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 381         defer func() {
 382                 os.RemoveAll(keepmount)
 383         }()
 384
 385         tmpdir, _ := ioutil.TempDir("", "")
 386         defer func() {
 387                 os.RemoveAll(tmpdir)
 388         }()
 389
 390         err := runner(ArvTestClient{c,
 391                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 392                 KeepTestClient{},
 393                 "zzzz-8i9sb-111111111111111",
 394                 "zzzz-ot0gb-111111111111111",
 395                 tmpdir,
 396                 keepmount,
 397                 Job{ScriptParameters: Tasks{[]TaskDef{{
 398                         Command: []string{"cat", "$(task.keep)/file1.txt"},
 399                         Stdout:  "output.txt"}}}},
 400                 Task{Sequence: 0})
 401         c.Check(err, IsNil)
 402
 403         checkOutput(c, tmpdir)
 404 }
 405
 406 func (s *TestSuite) TestSignal(c *C) {
 407         tmpdir, _ := ioutil.TempDir("", "")
 408         defer func() {
 409                 os.RemoveAll(tmpdir)
 410         }()
 411
 412         go func() {
 413                 time.Sleep(1 * time.Second)
 414                 self, _ := os.FindProcess(os.Getpid())
 415                 self.Signal(syscall.SIGINT)
 416         }()
 417
 418         err := runner(ArvTestClient{c,
 419                 "", false},
 420                 KeepTestClient{},
 421                 "zzzz-8i9sb-111111111111111",
 422                 "zzzz-ot0gb-111111111111111",
 423                 tmpdir,
 424                 "",
 425                 Job{ScriptParameters: Tasks{[]TaskDef{{
 426                         Command: []string{"sleep", "4"}}}}},
 427                 Task{Sequence: 0})
 428         c.Check(err, FitsTypeOf, PermFail{})
 429
 430 }
 431
 432 func (s *TestSuite) TestQuoting(c *C) {
 433         tmpdir, _ := ioutil.TempDir("", "")
 434         defer func() {
 435                 os.RemoveAll(tmpdir)
 436         }()
 437
 438         err := runner(ArvTestClient{c,
 439                 "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
 440                 KeepTestClient{},
 441                 "zzzz-8i9sb-111111111111111",
 442                 "zzzz-ot0gb-111111111111111",
 443                 tmpdir,
 444                 "",
 445                 Job{ScriptParameters: Tasks{[]TaskDef{{
 446                         Command: []string{"echo", "foo"},
 447                         Stdout:  "s ub:dir/:e vi\nl"}}}},
 448                 Task{Sequence: 0})
 449         c.Check(err, IsNil)
 450 }
 451
 452 func (s *TestSuite) TestKeepTmp(c *C) {
 453         tmpdir, _ := ioutil.TempDir("", "")
 454         defer func() {
 455                 os.RemoveAll(tmpdir)
 456         }()
 457
 458         os.Setenv("TASK_KEEPMOUNT_TMP", tmpdir)
 459         defer os.Setenv("TASK_KEEPMOUNT_TMP", "")
 460
 461         fn, err := os.Create(tmpdir + "/.arvados#collection")
 462         fn.Write([]byte("{\"manifest_text\":\". unparsed 0:3:foo\\n\",\"uuid\":null}"))
 463         defer fn.Close()
 464
 465         err = runner(ArvTestClient{c,
 466                 ". unparsed 0:3:foo\n", true},
 467                 KeepTestClient{},
 468                 "zzzz-8i9sb-111111111111111",
 469                 "zzzz-ot0gb-111111111111111",
 470                 tmpdir,
 471                 "",
 472                 Job{ScriptParameters: Tasks{[]TaskDef{{
 473                         Command:       []string{"echo", "foo"},
 474                         KeepTmpOutput: true}}}},
 475                 Task{Sequence: 0})
 476         c.Check(err, IsNil)
 477
 478 }