sdk/go/crunchrunner/crunchrunner_test.go

   1 package main
   2
   3 import (
   4         "io"
   5         "io/ioutil"
   6         "log"
   7         "os"
   8         "syscall"
   9         "testing"
  10         "time"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  13         . "gopkg.in/check.v1"
  14 )
  15
  16 // Gocheck boilerplate
  17 func Test(t *testing.T) {
  18         TestingT(t)
  19 }
  20
  21 type TestSuite struct{}
  22
  23 // Gocheck boilerplate
  24 var _ = Suite(&TestSuite{})
  25
  26 type ArvTestClient struct {
  27         c        *C
  28         manifest string
  29         success  bool
  30 }
  31
  32 func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
  33         return nil
  34 }
  35
  36 func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
  37         t.c.Check(resourceType, Equals, "job_tasks")
  38         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
  39                 Output:   t.manifest,
  40                 Success:  t.success,
  41                 Progress: 1}})
  42         return nil
  43 }
  44
  45 func (s *TestSuite) TestSimpleRun(c *C) {
  46         tmpdir, _ := ioutil.TempDir("", "")
  47         defer func() {
  48                 os.RemoveAll(tmpdir)
  49         }()
  50
  51         err := runner(ArvTestClient{c, "", true},
  52                 KeepTestClient{},
  53                 "zzzz-8i9sb-111111111111111",
  54                 "zzzz-ot0gb-111111111111111",
  55                 tmpdir,
  56                 "",
  57                 Job{ScriptParameters: Tasks{[]TaskDef{{
  58                         Command: []string{"echo", "foo"}}}}},
  59                 Task{Sequence: 0})
  60         c.Check(err, IsNil)
  61 }
  62
  63 func checkOutput(c *C, tmpdir string) {
  64         file, err := os.Open(tmpdir + "/outdir/output.txt")
  65         c.Assert(err, IsNil)
  66
  67         data := make([]byte, 100)
  68         var count int
  69         err = nil
  70         offset := 0
  71         for err == nil {
  72                 count, err = file.Read(data[offset:])
  73                 offset += count
  74         }
  75         c.Assert(err, Equals, io.EOF)
  76         c.Check(string(data[0:offset]), Equals, "foo\n")
  77 }
  78
  79 func (s *TestSuite) TestSimpleRunSubtask(c *C) {
  80         tmpdir, _ := ioutil.TempDir("", "")
  81         defer func() {
  82                 os.RemoveAll(tmpdir)
  83         }()
  84
  85         err := runner(ArvTestClient{c,
  86                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
  87                 KeepTestClient{},
  88                 "zzzz-8i9sb-111111111111111",
  89                 "zzzz-ot0gb-111111111111111",
  90                 tmpdir,
  91                 "",
  92                 Job{ScriptParameters: Tasks{[]TaskDef{
  93                         {Command: []string{"echo", "bar"}},
  94                         {Command: []string{"echo", "foo"}}}}},
  95                 Task{Parameters: TaskDef{
  96                         Command: []string{"echo", "foo"},
  97                         Stdout:  "output.txt"},
  98                         Sequence: 1})
  99         c.Check(err, IsNil)
 100
 101         checkOutput(c, tmpdir)
 102 }
 103
 104 func (s *TestSuite) TestRedirect(c *C) {
 105         tmpfile, _ := ioutil.TempFile("", "")
 106         tmpfile.Write([]byte("foo\n"))
 107         tmpfile.Close()
 108         defer os.Remove(tmpfile.Name())
 109
 110         tmpdir, _ := ioutil.TempDir("", "")
 111         defer func() {
 112                 os.RemoveAll(tmpdir)
 113         }()
 114
 115         err := runner(ArvTestClient{c,
 116                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 117                 KeepTestClient{},
 118                 "zzzz-8i9sb-111111111111111",
 119                 "zzzz-ot0gb-111111111111111",
 120                 tmpdir,
 121                 "",
 122                 Job{ScriptParameters: Tasks{[]TaskDef{{
 123                         Command: []string{"cat"},
 124                         Stdout:  "output.txt",
 125                         Stdin:   tmpfile.Name()}}}},
 126                 Task{Sequence: 0})
 127         c.Check(err, IsNil)
 128
 129         checkOutput(c, tmpdir)
 130 }
 131
 132 func (s *TestSuite) TestEnv(c *C) {
 133         tmpdir, _ := ioutil.TempDir("", "")
 134         defer func() {
 135                 os.RemoveAll(tmpdir)
 136         }()
 137
 138         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 139                 KeepTestClient{},
 140                 "zzzz-8i9sb-111111111111111",
 141                 "zzzz-ot0gb-111111111111111",
 142                 tmpdir,
 143                 "",
 144                 Job{ScriptParameters: Tasks{[]TaskDef{{
 145                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 146                         Stdout:  "output.txt",
 147                         Env:     map[string]string{"BAR": "foo"}}}}},
 148                 Task{Sequence: 0})
 149         c.Check(err, IsNil)
 150         checkOutput(c, tmpdir)
 151 }
 152
 153 func (s *TestSuite) TestEnvSubstitute(c *C) {
 154         tmpdir, _ := ioutil.TempDir("", "")
 155         defer func() {
 156                 os.RemoveAll(tmpdir)
 157         }()
 158
 159         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 160                 KeepTestClient{},
 161                 "zzzz-8i9sb-111111111111111",
 162                 "zzzz-ot0gb-111111111111111",
 163                 tmpdir,
 164                 "foo\n",
 165                 Job{ScriptParameters: Tasks{[]TaskDef{{
 166                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 167                         Stdout:  "output.txt",
 168                         Env:     map[string]string{"BAR": "$(task.keep)"}}}}},
 169                 Task{Sequence: 0})
 170         c.Check(err, IsNil)
 171         checkOutput(c, tmpdir)
 172 }
 173
 174 func (s *TestSuite) TestEnvReplace(c *C) {
 175         tmpdir, _ := ioutil.TempDir("", "")
 176         defer func() {
 177                 os.RemoveAll(tmpdir)
 178         }()
 179
 180         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 181                 KeepTestClient{},
 182                 "zzzz-8i9sb-111111111111111",
 183                 "zzzz-ot0gb-111111111111111",
 184                 tmpdir,
 185                 "",
 186                 Job{ScriptParameters: Tasks{[]TaskDef{{
 187                         Command: []string{"/bin/sh", "-c", "echo $PATH"},
 188                         Stdout:  "output.txt",
 189                         Env:     map[string]string{"PATH": "foo"}}}}},
 190                 Task{Sequence: 0})
 191         c.Check(err, IsNil)
 192         checkOutput(c, tmpdir)
 193 }
 194
 195 type SubtaskTestClient struct {
 196         c     *C
 197         parms []Task
 198         i     int
 199 }
 200
 201 func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
 202         t.c.Check(resourceType, Equals, "job_tasks")
 203         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": t.parms[t.i]})
 204         t.i += 1
 205         return nil
 206 }
 207
 208 func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 209         return nil
 210 }
 211
 212 func (s *TestSuite) TestScheduleSubtask(c *C) {
 213
 214         api := SubtaskTestClient{c, []Task{
 215                 {JobUUID: "zzzz-8i9sb-111111111111111",
 216                         CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
 217                         Sequence:             1,
 218                         Parameters: TaskDef{
 219                                 Command: []string{"echo", "bar"}}},
 220                 {JobUUID: "zzzz-8i9sb-111111111111111",
 221                         CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
 222                         Sequence:             1,
 223                         Parameters: TaskDef{
 224                                 Command: []string{"echo", "foo"}}}},
 225                 0}
 226
 227         tmpdir, _ := ioutil.TempDir("", "")
 228         defer func() {
 229                 os.RemoveAll(tmpdir)
 230         }()
 231
 232         err := runner(&api, KeepTestClient{},
 233                 "zzzz-8i9sb-111111111111111",
 234                 "zzzz-ot0gb-111111111111111",
 235                 tmpdir,
 236                 "",
 237                 Job{ScriptParameters: Tasks{[]TaskDef{
 238                         {Command: []string{"echo", "bar"}},
 239                         {Command: []string{"echo", "foo"}}}}},
 240                 Task{Sequence: 0})
 241         c.Check(err, IsNil)
 242
 243 }
 244
 245 func (s *TestSuite) TestRunFail(c *C) {
 246         tmpdir, _ := ioutil.TempDir("", "")
 247         defer func() {
 248                 os.RemoveAll(tmpdir)
 249         }()
 250
 251         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 252                 "zzzz-8i9sb-111111111111111",
 253                 "zzzz-ot0gb-111111111111111",
 254                 tmpdir,
 255                 "",
 256                 Job{ScriptParameters: Tasks{[]TaskDef{{
 257                         Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
 258                 Task{Sequence: 0})
 259         c.Check(err, FitsTypeOf, PermFail{})
 260 }
 261
 262 func (s *TestSuite) TestRunSuccessCode(c *C) {
 263         tmpdir, _ := ioutil.TempDir("", "")
 264         defer func() {
 265                 os.RemoveAll(tmpdir)
 266         }()
 267
 268         err := runner(ArvTestClient{c, "", true}, KeepTestClient{},
 269                 "zzzz-8i9sb-111111111111111",
 270                 "zzzz-ot0gb-111111111111111",
 271                 tmpdir,
 272                 "",
 273                 Job{ScriptParameters: Tasks{[]TaskDef{{
 274                         Command:      []string{"/bin/sh", "-c", "exit 1"},
 275                         SuccessCodes: []int{0, 1}}}}},
 276                 Task{Sequence: 0})
 277         c.Check(err, IsNil)
 278 }
 279
 280 func (s *TestSuite) TestRunFailCode(c *C) {
 281         tmpdir, _ := ioutil.TempDir("", "")
 282         defer func() {
 283                 os.RemoveAll(tmpdir)
 284         }()
 285
 286         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 287                 "zzzz-8i9sb-111111111111111",
 288                 "zzzz-ot0gb-111111111111111",
 289                 tmpdir,
 290                 "",
 291                 Job{ScriptParameters: Tasks{[]TaskDef{{
 292                         Command:            []string{"/bin/sh", "-c", "exit 0"},
 293                         PermanentFailCodes: []int{0, 1}}}}},
 294                 Task{Sequence: 0})
 295         c.Check(err, FitsTypeOf, PermFail{})
 296 }
 297
 298 func (s *TestSuite) TestRunTempFailCode(c *C) {
 299         tmpdir, _ := ioutil.TempDir("", "")
 300         defer func() {
 301                 os.RemoveAll(tmpdir)
 302         }()
 303
 304         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 305                 "zzzz-8i9sb-111111111111111",
 306                 "zzzz-ot0gb-111111111111111",
 307                 tmpdir,
 308                 "",
 309                 Job{ScriptParameters: Tasks{[]TaskDef{{
 310                         Command:            []string{"/bin/sh", "-c", "exit 1"},
 311                         TemporaryFailCodes: []int{1}}}}},
 312                 Task{Sequence: 0})
 313         c.Check(err, FitsTypeOf, TempFail{})
 314 }
 315
 316 func (s *TestSuite) TestVwd(c *C) {
 317         tmpfile, _ := ioutil.TempFile("", "")
 318         tmpfile.Write([]byte("foo\n"))
 319         tmpfile.Close()
 320         defer os.Remove(tmpfile.Name())
 321
 322         tmpdir, _ := ioutil.TempDir("", "")
 323         defer func() {
 324                 os.RemoveAll(tmpdir)
 325         }()
 326
 327         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 328                 KeepTestClient{},
 329                 "zzzz-8i9sb-111111111111111",
 330                 "zzzz-ot0gb-111111111111111",
 331                 tmpdir,
 332                 "",
 333                 Job{ScriptParameters: Tasks{[]TaskDef{{
 334                         Command: []string{"ls", "output.txt"},
 335                         Vwd: map[string]string{
 336                                 "output.txt": tmpfile.Name()}}}}},
 337                 Task{Sequence: 0})
 338         c.Check(err, IsNil)
 339         checkOutput(c, tmpdir)
 340 }
 341
 342 func (s *TestSuite) TestSubstitutionStdin(c *C) {
 343         keepmount, _ := ioutil.TempDir("", "")
 344         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 345         defer func() {
 346                 os.RemoveAll(keepmount)
 347         }()
 348
 349         log.Print("Keepmount is ", keepmount)
 350
 351         tmpdir, _ := ioutil.TempDir("", "")
 352         defer func() {
 353                 os.RemoveAll(tmpdir)
 354         }()
 355
 356         log.Print("tmpdir is ", tmpdir)
 357
 358         err := runner(ArvTestClient{c,
 359                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 360                 KeepTestClient{},
 361                 "zzzz-8i9sb-111111111111111",
 362                 "zzzz-ot0gb-111111111111111",
 363                 tmpdir,
 364                 keepmount,
 365                 Job{ScriptParameters: Tasks{[]TaskDef{{
 366                         Command: []string{"cat"},
 367                         Stdout:  "output.txt",
 368                         Stdin:   "$(task.keep)/file1.txt"}}}},
 369                 Task{Sequence: 0})
 370         c.Check(err, IsNil)
 371         checkOutput(c, tmpdir)
 372 }
 373
 374 func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
 375         keepmount, _ := ioutil.TempDir("", "")
 376         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 377         defer func() {
 378                 os.RemoveAll(keepmount)
 379         }()
 380
 381         tmpdir, _ := ioutil.TempDir("", "")
 382         defer func() {
 383                 os.RemoveAll(tmpdir)
 384         }()
 385
 386         err := runner(ArvTestClient{c,
 387                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 388                 KeepTestClient{},
 389                 "zzzz-8i9sb-111111111111111",
 390                 "zzzz-ot0gb-111111111111111",
 391                 tmpdir,
 392                 keepmount,
 393                 Job{ScriptParameters: Tasks{[]TaskDef{{
 394                         Command: []string{"cat", "$(task.keep)/file1.txt"},
 395                         Stdout:  "output.txt"}}}},
 396                 Task{Sequence: 0})
 397         c.Check(err, IsNil)
 398
 399         checkOutput(c, tmpdir)
 400 }
 401
 402 func (s *TestSuite) TestSignal(c *C) {
 403         tmpdir, _ := ioutil.TempDir("", "")
 404         defer func() {
 405                 os.RemoveAll(tmpdir)
 406         }()
 407
 408         go func() {
 409                 time.Sleep(1 * time.Second)
 410                 self, _ := os.FindProcess(os.Getpid())
 411                 self.Signal(syscall.SIGINT)
 412         }()
 413
 414         err := runner(ArvTestClient{c,
 415                 "", false},
 416                 KeepTestClient{},
 417                 "zzzz-8i9sb-111111111111111",
 418                 "zzzz-ot0gb-111111111111111",
 419                 tmpdir,
 420                 "",
 421                 Job{ScriptParameters: Tasks{[]TaskDef{{
 422                         Command: []string{"sleep", "4"}}}}},
 423                 Task{Sequence: 0})
 424         c.Check(err, FitsTypeOf, PermFail{})
 425
 426 }
 427
 428 func (s *TestSuite) TestQuoting(c *C) {
 429         tmpdir, _ := ioutil.TempDir("", "")
 430         defer func() {
 431                 os.RemoveAll(tmpdir)
 432         }()
 433
 434         err := runner(ArvTestClient{c,
 435                 "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
 436                 KeepTestClient{},
 437                 "zzzz-8i9sb-111111111111111",
 438                 "zzzz-ot0gb-111111111111111",
 439                 tmpdir,
 440                 "",
 441                 Job{ScriptParameters: Tasks{[]TaskDef{{
 442                         Command: []string{"echo", "foo"},
 443                         Stdout:  "s ub:dir/:e vi\nl"}}}},
 444                 Task{Sequence: 0})
 445         c.Check(err, IsNil)
 446 }
 447
 448 func (s *TestSuite) TestKeepTmp(c *C) {
 449         tmpdir, _ := ioutil.TempDir("", "")
 450         defer func() {
 451                 os.RemoveAll(tmpdir)
 452         }()
 453
 454         os.Setenv("TASK_KEEPMOUNT_TMP", tmpdir)
 455         defer os.Setenv("TASK_KEEPMOUNT_TMP", "")
 456
 457         fn, err := os.Create(tmpdir + "/.arvados#collection")
 458         fn.Write([]byte("{\"manifest_text\":\". unparsed 0:3:foo\\n\",\"uuid\":null}"))
 459         defer fn.Close()
 460
 461         err = runner(ArvTestClient{c,
 462                 ". unparsed 0:3:foo\n", true},
 463                 KeepTestClient{},
 464                 "zzzz-8i9sb-111111111111111",
 465                 "zzzz-ot0gb-111111111111111",
 466                 tmpdir,
 467                 "",
 468                 Job{ScriptParameters: Tasks{[]TaskDef{{
 469                         Command:       []string{"echo", "foo"},
 470                         KeepTmpOutput: true}}}},
 471                 Task{Sequence: 0})
 472         c.Check(err, IsNil)
 473
 474 }