services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/rand"
  10         "io/ioutil"
  11         "os"
  12         "time"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  17         "git.arvados.org/arvados.git/sdk/go/keepclient"
  18         "github.com/AdRoll/goamz/aws"
  19         "github.com/AdRoll/goamz/s3"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 func (s *IntegrationSuite) s3setup(c *check.C) (*arvados.Client, arvados.Collection, *s3.Bucket) {
  24         var coll arvados.Collection
  25         arv := arvados.NewClientFromEnv()
  26         arv.AuthToken = arvadostest.ActiveToken
  27         err := arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
  28                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
  29         }})
  30         c.Assert(err, check.IsNil)
  31         ac, err := arvadosclient.New(arv)
  32         c.Assert(err, check.IsNil)
  33         kc, err := keepclient.MakeKeepClient(ac)
  34         c.Assert(err, check.IsNil)
  35         fs, err := coll.FileSystem(arv, kc)
  36         c.Assert(err, check.IsNil)
  37         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
  38         c.Assert(err, check.IsNil)
  39         _, err = f.Write([]byte("⛵\n"))
  40         c.Assert(err, check.IsNil)
  41         err = f.Close()
  42         c.Assert(err, check.IsNil)
  43         err = fs.Sync()
  44         c.Assert(err, check.IsNil)
  45
  46         auth := aws.NewAuth(arvadostest.ActiveTokenV2, arvadostest.ActiveTokenV2, "", time.Now().Add(time.Hour))
  47         region := aws.Region{
  48                 Name:       s.testServer.Addr,
  49                 S3Endpoint: "http://" + s.testServer.Addr,
  50         }
  51         client := s3.New(*auth, region)
  52         bucket := &s3.Bucket{
  53                 S3:   client,
  54                 Name: coll.UUID,
  55         }
  56         return arv, coll, bucket
  57 }
  58
  59 func (s *IntegrationSuite) s3teardown(c *check.C, arv *arvados.Client, coll arvados.Collection) {
  60         err := arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
  61         c.Check(err, check.IsNil)
  62 }
  63
  64 func (s *IntegrationSuite) TestS3GetObject(c *check.C) {
  65         arv, coll, bucket := s.s3setup(c)
  66         defer s.s3teardown(c, arv, coll)
  67
  68         rdr, err := bucket.GetReader("emptyfile")
  69         c.Assert(err, check.IsNil)
  70         buf, err := ioutil.ReadAll(rdr)
  71         c.Check(err, check.IsNil)
  72         c.Check(len(buf), check.Equals, 0)
  73         err = rdr.Close()
  74         c.Check(err, check.IsNil)
  75
  76         rdr, err = bucket.GetReader("missingfile")
  77         c.Check(err, check.NotNil)
  78
  79         rdr, err = bucket.GetReader("sailboat.txt")
  80         c.Check(err, check.IsNil)
  81         buf, err = ioutil.ReadAll(rdr)
  82         c.Check(err, check.IsNil)
  83         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
  84         err = rdr.Close()
  85         c.Check(err, check.IsNil)
  86 }
  87
  88 func (s *IntegrationSuite) TestS3PutObjectSuccess(c *check.C) {
  89         arv, coll, bucket := s.s3setup(c)
  90         defer s.s3teardown(c, arv, coll)
  91
  92         for _, trial := range []struct {
  93                 objname string
  94                 size    int
  95         }{
  96                 {
  97                         objname: "newfile",
  98                         size:    128000000,
  99                 }, {
 100                         objname: "newdir/newfile",
 101                         size:    1 << 26,
 102                 }, {
 103                         objname: "newdir1/newdir2/newfile",
 104                         size:    0,
 105                 },
 106         } {
 107                 c.Logf("=== %v", trial)
 108
 109                 _, err := bucket.GetReader(trial.objname)
 110                 c.Assert(err, check.NotNil)
 111
 112                 buf := make([]byte, trial.size)
 113                 rand.Read(buf)
 114
 115                 err = bucket.PutReader(trial.objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 116                 c.Check(err, check.IsNil)
 117
 118                 rdr, err := bucket.GetReader(trial.objname)
 119                 if !c.Check(err, check.IsNil) {
 120                         continue
 121                 }
 122                 buf2, err := ioutil.ReadAll(rdr)
 123                 c.Check(err, check.IsNil)
 124                 c.Check(buf2, check.HasLen, len(buf))
 125                 c.Check(buf2, check.DeepEquals, buf)
 126         }
 127 }
 128
 129 func (s *IntegrationSuite) TestS3PutObjectFailure(c *check.C) {
 130         arv, coll, bucket := s.s3setup(c)
 131         defer s.s3teardown(c, arv, coll)
 132
 133         for _, trial := range []struct {
 134                 objname string
 135         }{
 136                 {
 137                         objname: "emptyfile/newname", // emptyfile exists, see s3setup()
 138                 }, {
 139                         objname: "emptyfile/", // emptyfile exists, see s3setup()
 140                 }, {
 141                         objname: "emptydir", // dir already exists, see s3setup()
 142                 }, {
 143                         objname: "emptydir/",
 144                 }, {
 145                         objname: "emptydir//",
 146                 }, {
 147                         objname: "newdir/",
 148                 }, {
 149                         objname: "newdir//",
 150                 }, {
 151                         objname: "/",
 152                 }, {
 153                         objname: "//",
 154                 }, {
 155                         objname: "foo//bar",
 156                 }, {
 157                         objname: "",
 158                 },
 159         } {
 160                 c.Logf("=== %v", trial)
 161
 162                 buf := make([]byte, 1234)
 163                 rand.Read(buf)
 164
 165                 err := bucket.PutReader(trial.objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 166                 if !c.Check(err, check.NotNil, check.Commentf("name %q should be rejected", trial.objname)) {
 167                         continue
 168                 }
 169
 170                 _, err = bucket.GetReader(trial.objname)
 171                 c.Check(err, check.NotNil)
 172         }
 173 }