services/crunch-run/upload_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "io/ioutil"
   9         "log"
  10         "os"
  11         "sync"
  12         "syscall"
  13
  14         . "gopkg.in/check.v1"
  15 )
  16
  17 type UploadTestSuite struct{}
  18
  19 // Gocheck boilerplate
  20 var _ = Suite(&UploadTestSuite{})
  21
  22 func (s *TestSuite) TestSimpleUpload(c *C) {
  23         tmpdir, _ := ioutil.TempDir("", "")
  24         defer func() {
  25                 os.RemoveAll(tmpdir)
  26         }()
  27
  28         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  29
  30         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  31         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
  32         c.Check(err, IsNil)
  33         c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
  34 }
  35
  36 func (s *TestSuite) TestSimpleUploadThreefiles(c *C) {
  37         tmpdir, _ := ioutil.TempDir("", "")
  38         defer func() {
  39                 os.RemoveAll(tmpdir)
  40         }()
  41
  42         for _, err := range []error{
  43                 ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600),
  44                 ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600),
  45                 os.Symlink("./file2.txt", tmpdir+"/file3.txt"),
  46                 syscall.Mkfifo(tmpdir+"/ignore.fifo", 0600),
  47         } {
  48                 c.Assert(err, IsNil)
  49         }
  50
  51         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  52         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
  53
  54         c.Check(err, IsNil)
  55         c.Check(str, Equals, ". aa65a413921163458c52fea478d5d3ee+9 0:3:file1.txt 3:3:file2.txt 6:3:file3.txt\n")
  56 }
  57
  58 func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
  59         tmpdir, _ := ioutil.TempDir("", "")
  60         defer func() {
  61                 os.RemoveAll(tmpdir)
  62         }()
  63
  64         os.Mkdir(tmpdir+"/subdir", 0700)
  65
  66         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  67         ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
  68
  69         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  70         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
  71
  72         c.Check(err, IsNil)
  73
  74         // streams can get added in either order because of scheduling
  75         // of goroutines.
  76         if str != `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
  77 ./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
  78 ` && str != `./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
  79 . acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
  80 ` {
  81                 c.Error("Did not get expected manifest text")
  82         }
  83 }
  84
  85 func (s *TestSuite) TestSimpleUploadLarge(c *C) {
  86         tmpdir, _ := ioutil.TempDir("", "")
  87         defer func() {
  88                 os.RemoveAll(tmpdir)
  89         }()
  90
  91         file, _ := os.Create(tmpdir + "/" + "file1.txt")
  92         data := make([]byte, 1024*1024-1)
  93         for i := range data {
  94                 data[i] = byte(i % 10)
  95         }
  96         for i := 0; i < 65; i++ {
  97                 file.Write(data)
  98         }
  99         file.Close()
 100
 101         ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
 102
 103         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 104         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
 105
 106         c.Check(err, IsNil)
 107         c.Check(str, Equals, ". 00ecf01e0d93385115c9f8bed757425d+67108864 485cd630387b6b1846fe429f261ea05f+1048514 0:68157375:file1.txt 68157375:3:file2.txt\n")
 108 }
 109
 110 func (s *TestSuite) TestUploadEmptySubdir(c *C) {
 111         tmpdir, _ := ioutil.TempDir("", "")
 112         defer func() {
 113                 os.RemoveAll(tmpdir)
 114         }()
 115
 116         os.Mkdir(tmpdir+"/subdir", 0700)
 117
 118         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 119
 120         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 121         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
 122
 123         c.Check(err, IsNil)
 124         c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
 125 `)
 126 }
 127
 128 func (s *TestSuite) TestUploadEmptyFile(c *C) {
 129         tmpdir, _ := ioutil.TempDir("", "")
 130         defer func() {
 131                 os.RemoveAll(tmpdir)
 132         }()
 133
 134         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
 135
 136         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 137         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
 138
 139         c.Check(err, IsNil)
 140         c.Check(str, Equals, `. d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1.txt
 141 `)
 142 }
 143
 144 func (s *TestSuite) TestUploadError(c *C) {
 145         tmpdir, _ := ioutil.TempDir("", "")
 146         defer func() {
 147                 os.RemoveAll(tmpdir)
 148         }()
 149
 150         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 151
 152         cw := CollectionWriter{0, &KeepErrorTestClient{}, nil, nil, sync.Mutex{}}
 153         str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
 154
 155         c.Check(err, NotNil)
 156         c.Check(str, Equals, "")
 157 }