sdk/go/crunchrunner/crunchrunner_test.go

   1 package main
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   5         . "gopkg.in/check.v1"
   6         "io"
   7         "io/ioutil"
   8         "log"
   9         "os"
  10         "syscall"
  11         "testing"
  12         "time"
  13 )
  14
  15 // Gocheck boilerplate
  16 func Test(t *testing.T) {
  17         TestingT(t)
  18 }
  19
  20 type TestSuite struct{}
  21
  22 // Gocheck boilerplate
  23 var _ = Suite(&TestSuite{})
  24
  25 type ArvTestClient struct {
  26         c        *C
  27         manifest string
  28         success  bool
  29 }
  30
  31 func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
  32         return nil
  33 }
  34
  35 func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
  36         t.c.Check(resourceType, Equals, "job_tasks")
  37         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
  38                 Output:   t.manifest,
  39                 Success:  t.success,
  40                 Progress: 1}})
  41         return nil
  42 }
  43
  44 func (s *TestSuite) TestSimpleRun(c *C) {
  45         tmpdir, _ := ioutil.TempDir("", "")
  46         defer func() {
  47                 os.RemoveAll(tmpdir)
  48         }()
  49
  50         err := runner(ArvTestClient{c, "", true},
  51                 KeepTestClient{},
  52                 "zzzz-8i9sb-111111111111111",
  53                 "zzzz-ot0gb-111111111111111",
  54                 tmpdir,
  55                 "",
  56                 Job{Script_parameters: Tasks{[]TaskDef{{
  57                         Command: []string{"echo", "foo"}}}}},
  58                 Task{Sequence: 0})
  59         c.Check(err, IsNil)
  60 }
  61
  62 func checkOutput(c *C, tmpdir string) {
  63         file, err := os.Open(tmpdir + "/outdir/output.txt")
  64         c.Assert(err, IsNil)
  65
  66         data := make([]byte, 100)
  67         var count int
  68         err = nil
  69         offset := 0
  70         for err == nil {
  71                 count, err = file.Read(data[offset:])
  72                 offset += count
  73         }
  74         c.Assert(err, Equals, io.EOF)
  75         c.Check(string(data[0:offset]), Equals, "foo\n")
  76 }
  77
  78 func (s *TestSuite) TestSimpleRunSubtask(c *C) {
  79         tmpdir, _ := ioutil.TempDir("", "")
  80         defer func() {
  81                 os.RemoveAll(tmpdir)
  82         }()
  83
  84         err := runner(ArvTestClient{c,
  85                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
  86                 KeepTestClient{},
  87                 "zzzz-8i9sb-111111111111111",
  88                 "zzzz-ot0gb-111111111111111",
  89                 tmpdir,
  90                 "",
  91                 Job{Script_parameters: Tasks{[]TaskDef{
  92                         {Command: []string{"echo", "bar"}},
  93                         {Command: []string{"echo", "foo"}}}}},
  94                 Task{Parameters: TaskDef{
  95                         Command: []string{"echo", "foo"},
  96                         Stdout:  "output.txt"},
  97                         Sequence: 1})
  98         c.Check(err, IsNil)
  99
 100         checkOutput(c, tmpdir)
 101 }
 102
 103 func (s *TestSuite) TestRedirect(c *C) {
 104         tmpfile, _ := ioutil.TempFile("", "")
 105         tmpfile.Write([]byte("foo\n"))
 106         tmpfile.Close()
 107         defer os.Remove(tmpfile.Name())
 108
 109         tmpdir, _ := ioutil.TempDir("", "")
 110         defer func() {
 111                 os.RemoveAll(tmpdir)
 112         }()
 113
 114         err := runner(ArvTestClient{c,
 115                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 116                 KeepTestClient{},
 117                 "zzzz-8i9sb-111111111111111",
 118                 "zzzz-ot0gb-111111111111111",
 119                 tmpdir,
 120                 "",
 121                 Job{Script_parameters: Tasks{[]TaskDef{{
 122                         Command: []string{"cat"},
 123                         Stdout:  "output.txt",
 124                         Stdin:   tmpfile.Name()}}}},
 125                 Task{Sequence: 0})
 126         c.Check(err, IsNil)
 127
 128         checkOutput(c, tmpdir)
 129 }
 130
 131 func (s *TestSuite) TestEnv(c *C) {
 132         tmpdir, _ := ioutil.TempDir("", "")
 133         defer func() {
 134                 os.RemoveAll(tmpdir)
 135         }()
 136
 137         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 138                 KeepTestClient{},
 139                 "zzzz-8i9sb-111111111111111",
 140                 "zzzz-ot0gb-111111111111111",
 141                 tmpdir,
 142                 "",
 143                 Job{Script_parameters: Tasks{[]TaskDef{{
 144                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 145                         Stdout:  "output.txt",
 146                         Env:     map[string]string{"BAR": "foo"}}}}},
 147                 Task{Sequence: 0})
 148         c.Check(err, IsNil)
 149         checkOutput(c, tmpdir)
 150 }
 151
 152 func (s *TestSuite) TestEnvSubstitute(c *C) {
 153         tmpdir, _ := ioutil.TempDir("", "")
 154         defer func() {
 155                 os.RemoveAll(tmpdir)
 156         }()
 157
 158         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 159                 KeepTestClient{},
 160                 "zzzz-8i9sb-111111111111111",
 161                 "zzzz-ot0gb-111111111111111",
 162                 tmpdir,
 163                 "foo\n",
 164                 Job{Script_parameters: Tasks{[]TaskDef{{
 165                         Command: []string{"/bin/sh", "-c", "echo $BAR"},
 166                         Stdout:  "output.txt",
 167                         Env:     map[string]string{"BAR": "$(task.keep)"}}}}},
 168                 Task{Sequence: 0})
 169         c.Check(err, IsNil)
 170         checkOutput(c, tmpdir)
 171 }
 172
 173 func (s *TestSuite) TestEnvReplace(c *C) {
 174         tmpdir, _ := ioutil.TempDir("", "")
 175         defer func() {
 176                 os.RemoveAll(tmpdir)
 177         }()
 178
 179         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 180                 KeepTestClient{},
 181                 "zzzz-8i9sb-111111111111111",
 182                 "zzzz-ot0gb-111111111111111",
 183                 tmpdir,
 184                 "",
 185                 Job{Script_parameters: Tasks{[]TaskDef{{
 186                         Command: []string{"/bin/sh", "-c", "echo $PATH"},
 187                         Stdout:  "output.txt",
 188                         Env:     map[string]string{"PATH": "foo"}}}}},
 189                 Task{Sequence: 0})
 190         c.Check(err, IsNil)
 191         checkOutput(c, tmpdir)
 192 }
 193
 194 type SubtaskTestClient struct {
 195         c     *C
 196         parms []Task
 197         i     int
 198 }
 199
 200 func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error {
 201         t.c.Check(resourceType, Equals, "job_tasks")
 202         t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": t.parms[t.i]})
 203         t.i += 1
 204         return nil
 205 }
 206
 207 func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 208         return nil
 209 }
 210
 211 func (s *TestSuite) TestScheduleSubtask(c *C) {
 212
 213         api := SubtaskTestClient{c, []Task{
 214                 {Job_uuid: "zzzz-8i9sb-111111111111111",
 215                         Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
 216                         Sequence:                 1,
 217                         Parameters: TaskDef{
 218                                 Command: []string{"echo", "bar"}}},
 219                 {Job_uuid: "zzzz-8i9sb-111111111111111",
 220                         Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
 221                         Sequence:                 1,
 222                         Parameters: TaskDef{
 223                                 Command: []string{"echo", "foo"}}}},
 224                 0}
 225
 226         tmpdir, _ := ioutil.TempDir("", "")
 227         defer func() {
 228                 os.RemoveAll(tmpdir)
 229         }()
 230
 231         err := runner(&api, KeepTestClient{},
 232                 "zzzz-8i9sb-111111111111111",
 233                 "zzzz-ot0gb-111111111111111",
 234                 tmpdir,
 235                 "",
 236                 Job{Script_parameters: Tasks{[]TaskDef{
 237                         {Command: []string{"echo", "bar"}},
 238                         {Command: []string{"echo", "foo"}}}}},
 239                 Task{Sequence: 0})
 240         c.Check(err, IsNil)
 241
 242 }
 243
 244 func (s *TestSuite) TestRunFail(c *C) {
 245         tmpdir, _ := ioutil.TempDir("", "")
 246         defer func() {
 247                 os.RemoveAll(tmpdir)
 248         }()
 249
 250         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 251                 "zzzz-8i9sb-111111111111111",
 252                 "zzzz-ot0gb-111111111111111",
 253                 tmpdir,
 254                 "",
 255                 Job{Script_parameters: Tasks{[]TaskDef{{
 256                         Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
 257                 Task{Sequence: 0})
 258         c.Check(err, FitsTypeOf, PermFail{})
 259 }
 260
 261 func (s *TestSuite) TestRunSuccessCode(c *C) {
 262         tmpdir, _ := ioutil.TempDir("", "")
 263         defer func() {
 264                 os.RemoveAll(tmpdir)
 265         }()
 266
 267         err := runner(ArvTestClient{c, "", true}, KeepTestClient{},
 268                 "zzzz-8i9sb-111111111111111",
 269                 "zzzz-ot0gb-111111111111111",
 270                 tmpdir,
 271                 "",
 272                 Job{Script_parameters: Tasks{[]TaskDef{{
 273                         Command:      []string{"/bin/sh", "-c", "exit 1"},
 274                         SuccessCodes: []int{0, 1}}}}},
 275                 Task{Sequence: 0})
 276         c.Check(err, IsNil)
 277 }
 278
 279 func (s *TestSuite) TestRunFailCode(c *C) {
 280         tmpdir, _ := ioutil.TempDir("", "")
 281         defer func() {
 282                 os.RemoveAll(tmpdir)
 283         }()
 284
 285         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 286                 "zzzz-8i9sb-111111111111111",
 287                 "zzzz-ot0gb-111111111111111",
 288                 tmpdir,
 289                 "",
 290                 Job{Script_parameters: Tasks{[]TaskDef{{
 291                         Command:            []string{"/bin/sh", "-c", "exit 0"},
 292                         PermanentFailCodes: []int{0, 1}}}}},
 293                 Task{Sequence: 0})
 294         c.Check(err, FitsTypeOf, PermFail{})
 295 }
 296
 297 func (s *TestSuite) TestRunTempFailCode(c *C) {
 298         tmpdir, _ := ioutil.TempDir("", "")
 299         defer func() {
 300                 os.RemoveAll(tmpdir)
 301         }()
 302
 303         err := runner(ArvTestClient{c, "", false}, KeepTestClient{},
 304                 "zzzz-8i9sb-111111111111111",
 305                 "zzzz-ot0gb-111111111111111",
 306                 tmpdir,
 307                 "",
 308                 Job{Script_parameters: Tasks{[]TaskDef{{
 309                         Command:            []string{"/bin/sh", "-c", "exit 1"},
 310                         TemporaryFailCodes: []int{1}}}}},
 311                 Task{Sequence: 0})
 312         c.Check(err, FitsTypeOf, TempFail{})
 313 }
 314
 315 func (s *TestSuite) TestVwd(c *C) {
 316         tmpfile, _ := ioutil.TempFile("", "")
 317         tmpfile.Write([]byte("foo\n"))
 318         tmpfile.Close()
 319         defer os.Remove(tmpfile.Name())
 320
 321         tmpdir, _ := ioutil.TempDir("", "")
 322         defer func() {
 323                 os.RemoveAll(tmpdir)
 324         }()
 325
 326         err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 327                 KeepTestClient{},
 328                 "zzzz-8i9sb-111111111111111",
 329                 "zzzz-ot0gb-111111111111111",
 330                 tmpdir,
 331                 "",
 332                 Job{Script_parameters: Tasks{[]TaskDef{{
 333                         Command: []string{"ls", "output.txt"},
 334                         Vwd: map[string]string{
 335                                 "output.txt": tmpfile.Name()}}}}},
 336                 Task{Sequence: 0})
 337         c.Check(err, IsNil)
 338         checkOutput(c, tmpdir)
 339 }
 340
 341 func (s *TestSuite) TestSubstitutionStdin(c *C) {
 342         keepmount, _ := ioutil.TempDir("", "")
 343         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 344         defer func() {
 345                 os.RemoveAll(keepmount)
 346         }()
 347
 348         log.Print("Keepmount is ", keepmount)
 349
 350         tmpdir, _ := ioutil.TempDir("", "")
 351         defer func() {
 352                 os.RemoveAll(tmpdir)
 353         }()
 354
 355         log.Print("tmpdir is ", tmpdir)
 356
 357         err := runner(ArvTestClient{c,
 358                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 359                 KeepTestClient{},
 360                 "zzzz-8i9sb-111111111111111",
 361                 "zzzz-ot0gb-111111111111111",
 362                 tmpdir,
 363                 keepmount,
 364                 Job{Script_parameters: Tasks{[]TaskDef{{
 365                         Command: []string{"cat"},
 366                         Stdout:  "output.txt",
 367                         Stdin:   "$(task.keep)/file1.txt"}}}},
 368                 Task{Sequence: 0})
 369         c.Check(err, IsNil)
 370         checkOutput(c, tmpdir)
 371 }
 372
 373 func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
 374         keepmount, _ := ioutil.TempDir("", "")
 375         ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
 376         defer func() {
 377                 os.RemoveAll(keepmount)
 378         }()
 379
 380         tmpdir, _ := ioutil.TempDir("", "")
 381         defer func() {
 382                 os.RemoveAll(tmpdir)
 383         }()
 384
 385         err := runner(ArvTestClient{c,
 386                 ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
 387                 KeepTestClient{},
 388                 "zzzz-8i9sb-111111111111111",
 389                 "zzzz-ot0gb-111111111111111",
 390                 tmpdir,
 391                 keepmount,
 392                 Job{Script_parameters: Tasks{[]TaskDef{{
 393                         Command: []string{"cat", "$(task.keep)/file1.txt"},
 394                         Stdout:  "output.txt"}}}},
 395                 Task{Sequence: 0})
 396         c.Check(err, IsNil)
 397
 398         checkOutput(c, tmpdir)
 399 }
 400
 401 func (s *TestSuite) TestSignal(c *C) {
 402         tmpdir, _ := ioutil.TempDir("", "")
 403         defer func() {
 404                 os.RemoveAll(tmpdir)
 405         }()
 406
 407         go func() {
 408                 time.Sleep(1 * time.Second)
 409                 self, _ := os.FindProcess(os.Getpid())
 410                 self.Signal(syscall.SIGINT)
 411         }()
 412
 413         err := runner(ArvTestClient{c,
 414                 "", false},
 415                 KeepTestClient{},
 416                 "zzzz-8i9sb-111111111111111",
 417                 "zzzz-ot0gb-111111111111111",
 418                 tmpdir,
 419                 "",
 420                 Job{Script_parameters: Tasks{[]TaskDef{{
 421                         Command: []string{"sleep", "4"}}}}},
 422                 Task{Sequence: 0})
 423         c.Check(err, FitsTypeOf, PermFail{})
 424
 425 }
 426
 427 func (s *TestSuite) TestQuoting(c *C) {
 428         tmpdir, _ := ioutil.TempDir("", "")
 429         defer func() {
 430                 os.RemoveAll(tmpdir)
 431         }()
 432
 433         err := runner(ArvTestClient{c,
 434                 "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
 435                 KeepTestClient{},
 436                 "zzzz-8i9sb-111111111111111",
 437                 "zzzz-ot0gb-111111111111111",
 438                 tmpdir,
 439                 "",
 440                 Job{Script_parameters: Tasks{[]TaskDef{{
 441                         Command: []string{"echo", "foo"},
 442                         Stdout:  "s ub:dir/:e vi\nl"}}}},
 443                 Task{Sequence: 0})
 444         c.Check(err, IsNil)
 445 }
 446
 447 func (s *TestSuite) TestKeepTmp(c *C) {
 448         tmpdir, _ := ioutil.TempDir("", "")
 449         defer func() {
 450                 os.RemoveAll(tmpdir)
 451         }()
 452
 453         os.Setenv("TASK_KEEPMOUNT_TMP", tmpdir)
 454         defer os.Setenv("TASK_KEEPMOUNT_TMP", "")
 455
 456         fn, err := os.Create(tmpdir + "/.arvados#collection")
 457         fn.Write([]byte("{\"manifest_text\":\". unparsed 0:3:foo\\n\",\"uuid\":null}"))
 458         defer fn.Close()
 459
 460         err = runner(ArvTestClient{c,
 461                 ". unparsed 0:3:foo\n", true},
 462                 KeepTestClient{},
 463                 "zzzz-8i9sb-111111111111111",
 464                 "zzzz-ot0gb-111111111111111",
 465                 tmpdir,
 466                 "",
 467                 Job{Script_parameters: Tasks{[]TaskDef{{
 468                         Command:       []string{"echo", "foo"},
 469                         KeepTmpOutput: true}}}},
 470                 Task{Sequence: 0})
 471         c.Check(err, IsNil)
 472
 473 }