services/crunch-run/upload_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         . "gopkg.in/check.v1"
   9         "io/ioutil"
  10         "log"
  11         "os"
  12         "path/filepath"
  13         "sync"
  14         "syscall"
  15 )
  16
  17 type UploadTestSuite struct{}
  18
  19 // Gocheck boilerplate
  20 var _ = Suite(&UploadTestSuite{})
  21
  22 func writeTree(cw *CollectionWriter, root string, status *log.Logger) (mt string, err error) {
  23         walkUpload := cw.BeginUpload(root, status)
  24
  25         err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
  26                 info, _ = os.Stat(path)
  27                 if info.Mode().IsRegular() {
  28                         return walkUpload.UploadFile(path, path)
  29                 }
  30                 return nil
  31         })
  32
  33         cw.EndUpload(walkUpload)
  34         if err != nil {
  35                 return "", err
  36         }
  37         mt, err = cw.ManifestText()
  38         return
  39 }
  40
  41 func (s *TestSuite) TestSimpleUpload(c *C) {
  42         tmpdir, _ := ioutil.TempDir("", "")
  43         defer func() {
  44                 os.RemoveAll(tmpdir)
  45         }()
  46
  47         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  48
  49         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  50         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  51         c.Check(err, IsNil)
  52         c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
  53 }
  54
  55 func (s *TestSuite) TestUploadThreeFiles(c *C) {
  56         tmpdir, _ := ioutil.TempDir("", "")
  57         defer func() {
  58                 os.RemoveAll(tmpdir)
  59         }()
  60
  61         for _, err := range []error{
  62                 ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600),
  63                 ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600),
  64                 os.Symlink("./file2.txt", tmpdir+"/file3.txt"),
  65                 syscall.Mkfifo(tmpdir+"/ignore.fifo", 0600),
  66         } {
  67                 c.Assert(err, IsNil)
  68         }
  69
  70         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  71         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  72
  73         c.Check(err, IsNil)
  74         c.Check(str, Equals, ". aa65a413921163458c52fea478d5d3ee+9 0:3:file1.txt 3:3:file2.txt 6:3:file3.txt\n")
  75 }
  76
  77 func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
  78         tmpdir, _ := ioutil.TempDir("", "")
  79         defer func() {
  80                 os.RemoveAll(tmpdir)
  81         }()
  82
  83         os.Mkdir(tmpdir+"/subdir", 0700)
  84
  85         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  86         ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
  87
  88         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
  89         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  90
  91         c.Check(err, IsNil)
  92
  93         // streams can get added in either order because of scheduling
  94         // of goroutines.
  95         if str != `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
  96 ./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
  97 ` && str != `./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
  98 . acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
  99 ` {
 100                 c.Error("Did not get expected manifest text")
 101         }
 102 }
 103
 104 func (s *TestSuite) TestSimpleUploadLarge(c *C) {
 105         tmpdir, _ := ioutil.TempDir("", "")
 106         defer func() {
 107                 os.RemoveAll(tmpdir)
 108         }()
 109
 110         file, _ := os.Create(tmpdir + "/" + "file1.txt")
 111         data := make([]byte, 1024*1024-1)
 112         for i := range data {
 113                 data[i] = byte(i % 10)
 114         }
 115         for i := 0; i < 65; i++ {
 116                 file.Write(data)
 117         }
 118         file.Close()
 119
 120         ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
 121
 122         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 123         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 124
 125         c.Check(err, IsNil)
 126         c.Check(str, Equals, ". 00ecf01e0d93385115c9f8bed757425d+67108864 485cd630387b6b1846fe429f261ea05f+1048514 0:68157375:file1.txt 68157375:3:file2.txt\n")
 127 }
 128
 129 func (s *TestSuite) TestUploadEmptySubdir(c *C) {
 130         tmpdir, _ := ioutil.TempDir("", "")
 131         defer func() {
 132                 os.RemoveAll(tmpdir)
 133         }()
 134
 135         os.Mkdir(tmpdir+"/subdir", 0700)
 136
 137         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 138
 139         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 140         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 141
 142         c.Check(err, IsNil)
 143         c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
 144 `)
 145 }
 146
 147 func (s *TestSuite) TestUploadEmptyFile(c *C) {
 148         tmpdir, _ := ioutil.TempDir("", "")
 149         defer func() {
 150                 os.RemoveAll(tmpdir)
 151         }()
 152
 153         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
 154
 155         cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
 156         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 157
 158         c.Check(err, IsNil)
 159         c.Check(str, Equals, `. d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1.txt
 160 `)
 161 }
 162
 163 func (s *TestSuite) TestUploadError(c *C) {
 164         tmpdir, _ := ioutil.TempDir("", "")
 165         defer func() {
 166                 os.RemoveAll(tmpdir)
 167         }()
 168
 169         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 170
 171         cw := CollectionWriter{0, &KeepErrorTestClient{}, nil, nil, sync.Mutex{}}
 172         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 173
 174         c.Check(err, NotNil)
 175         c.Check(str, Equals, "")
 176 }