services/crunch-run/upload_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "io/ioutil"
   9         "log"
  10         "os"
  11         "path/filepath"
  12         "sync"
  13         "syscall"
  14
  15         . "gopkg.in/check.v1"
  16 )
  17
  18 type UploadTestSuite struct{}
  19
  20 // Gocheck boilerplate
  21 var _ = Suite(&UploadTestSuite{})
  22
  23 func writeTree(cw *CollectionWriter, root string, status *log.Logger) (mt string, err error) {
  24         walkUpload := cw.BeginUpload(root, status)
  25
  26         err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
  27                 info, _ = os.Stat(path)
  28                 if info.Mode().IsRegular() {
  29                         return walkUpload.UploadFile(path, path)
  30                 }
  31                 return nil
  32         })
  33
  34         cw.EndUpload(walkUpload)
  35         if err != nil {
  36                 return "", err
  37         }
  38         mt, err = cw.ManifestText()
  39         return
  40 }
  41
  42 func (s *TestSuite) TestSimpleUpload(c *C) {
  43         tmpdir, _ := ioutil.TempDir("", "")
  44         defer func() {
  45                 os.RemoveAll(tmpdir)
  46         }()
  47
  48         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  49
  50         kc := &KeepTestClient{}
  51         defer kc.Close()
  52         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
  53         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  54         c.Check(err, IsNil)
  55         c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
  56 }
  57
  58 func (s *TestSuite) TestUploadThreeFiles(c *C) {
  59         tmpdir, _ := ioutil.TempDir("", "")
  60         defer func() {
  61                 os.RemoveAll(tmpdir)
  62         }()
  63
  64         for _, err := range []error{
  65                 ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600),
  66                 ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600),
  67                 os.Symlink("./file2.txt", tmpdir+"/file3.txt"),
  68                 syscall.Mkfifo(tmpdir+"/ignore.fifo", 0600),
  69         } {
  70                 c.Assert(err, IsNil)
  71         }
  72
  73         kc := &KeepTestClient{}
  74         defer kc.Close()
  75         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
  76         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  77
  78         c.Check(err, IsNil)
  79         c.Check(str, Equals, ". aa65a413921163458c52fea478d5d3ee+9 0:3:file1.txt 3:3:file2.txt 6:3:file3.txt\n")
  80 }
  81
  82 func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
  83         tmpdir, _ := ioutil.TempDir("", "")
  84         defer func() {
  85                 os.RemoveAll(tmpdir)
  86         }()
  87
  88         os.Mkdir(tmpdir+"/subdir", 0700)
  89
  90         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
  91         ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
  92
  93         kc := &KeepTestClient{}
  94         defer kc.Close()
  95         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
  96         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
  97
  98         c.Check(err, IsNil)
  99
 100         // streams can get added in either order because of scheduling
 101         // of goroutines.
 102         if str != `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
 103 ./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
 104 ` && str != `./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
 105 . acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
 106 ` {
 107                 c.Error("Did not get expected manifest text")
 108         }
 109 }
 110
 111 func (s *TestSuite) TestSimpleUploadLarge(c *C) {
 112         tmpdir, _ := ioutil.TempDir("", "")
 113         defer func() {
 114                 os.RemoveAll(tmpdir)
 115         }()
 116
 117         file, _ := os.Create(tmpdir + "/" + "file1.txt")
 118         data := make([]byte, 1024*1024-1)
 119         for i := range data {
 120                 data[i] = byte(i % 10)
 121         }
 122         for i := 0; i < 65; i++ {
 123                 file.Write(data)
 124         }
 125         file.Close()
 126
 127         ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
 128
 129         kc := &KeepTestClient{}
 130         defer kc.Close()
 131         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
 132         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 133
 134         c.Check(err, IsNil)
 135         c.Check(str, Equals, ". 00ecf01e0d93385115c9f8bed757425d+67108864 485cd630387b6b1846fe429f261ea05f+1048514 0:68157375:file1.txt 68157375:3:file2.txt\n")
 136 }
 137
 138 func (s *TestSuite) TestUploadEmptySubdir(c *C) {
 139         tmpdir, _ := ioutil.TempDir("", "")
 140         defer func() {
 141                 os.RemoveAll(tmpdir)
 142         }()
 143
 144         os.Mkdir(tmpdir+"/subdir", 0700)
 145
 146         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 147
 148         kc := &KeepTestClient{}
 149         defer kc.Close()
 150         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
 151         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 152
 153         c.Check(err, IsNil)
 154         c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
 155 `)
 156 }
 157
 158 func (s *TestSuite) TestUploadEmptyFile(c *C) {
 159         tmpdir, _ := ioutil.TempDir("", "")
 160         defer func() {
 161                 os.RemoveAll(tmpdir)
 162         }()
 163
 164         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
 165
 166         kc := &KeepTestClient{}
 167         defer kc.Close()
 168         cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
 169         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 170
 171         c.Check(err, IsNil)
 172         c.Check(str, Equals, `. d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1.txt
 173 `)
 174 }
 175
 176 func (s *TestSuite) TestUploadError(c *C) {
 177         tmpdir, _ := ioutil.TempDir("", "")
 178         defer func() {
 179                 os.RemoveAll(tmpdir)
 180         }()
 181
 182         ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
 183
 184         cw := CollectionWriter{0, &KeepErrorTestClient{}, nil, nil, sync.Mutex{}}
 185         str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
 186
 187         c.Check(err, NotNil)
 188         c.Check(str, Equals, "")
 189 }