services/keep-web/s3_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/rand"
  10         "fmt"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "os/exec"
  17         "strings"
  18         "sync"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/keepclient"
  25         "github.com/AdRoll/goamz/aws"
  26         "github.com/AdRoll/goamz/s3"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 type s3stage struct {
  31         arv        *arvados.Client
  32         ac         *arvadosclient.ArvadosClient
  33         kc         *keepclient.KeepClient
  34         proj       arvados.Group
  35         projbucket *s3.Bucket
  36         coll       arvados.Collection
  37         collbucket *s3.Bucket
  38 }
  39
  40 func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
  41         var proj arvados.Group
  42         var coll arvados.Collection
  43         arv := arvados.NewClientFromEnv()
  44         arv.AuthToken = arvadostest.ActiveToken
  45         err := arv.RequestAndDecode(&proj, "POST", "arvados/v1/groups", nil, map[string]interface{}{
  46                 "group": map[string]interface{}{
  47                         "group_class": "project",
  48                         "name":        "keep-web s3 test",
  49                 },
  50                 "ensure_unique_name": true,
  51         })
  52         c.Assert(err, check.IsNil)
  53         err = arv.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{
  54                 "owner_uuid":    proj.UUID,
  55                 "name":          "keep-web s3 test collection",
  56                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:emptyfile\n./emptydir d41d8cd98f00b204e9800998ecf8427e+0 0:0:.\n",
  57         }})
  58         c.Assert(err, check.IsNil)
  59         ac, err := arvadosclient.New(arv)
  60         c.Assert(err, check.IsNil)
  61         kc, err := keepclient.MakeKeepClient(ac)
  62         c.Assert(err, check.IsNil)
  63         fs, err := coll.FileSystem(arv, kc)
  64         c.Assert(err, check.IsNil)
  65         f, err := fs.OpenFile("sailboat.txt", os.O_CREATE|os.O_WRONLY, 0644)
  66         c.Assert(err, check.IsNil)
  67         _, err = f.Write([]byte("⛵\n"))
  68         c.Assert(err, check.IsNil)
  69         err = f.Close()
  70         c.Assert(err, check.IsNil)
  71         err = fs.Sync()
  72         c.Assert(err, check.IsNil)
  73         err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
  74         c.Assert(err, check.IsNil)
  75
  76         auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
  77         region := aws.Region{
  78                 Name:       s.testServer.Addr,
  79                 S3Endpoint: "http://" + s.testServer.Addr,
  80         }
  81         client := s3.New(*auth, region)
  82         client.Signature = aws.V4Signature
  83         return s3stage{
  84                 arv:  arv,
  85                 ac:   ac,
  86                 kc:   kc,
  87                 proj: proj,
  88                 projbucket: &s3.Bucket{
  89                         S3:   client,
  90                         Name: proj.UUID,
  91                 },
  92                 coll: coll,
  93                 collbucket: &s3.Bucket{
  94                         S3:   client,
  95                         Name: coll.UUID,
  96                 },
  97         }
  98 }
  99
 100 func (stage s3stage) teardown(c *check.C) {
 101         if stage.coll.UUID != "" {
 102                 err := stage.arv.RequestAndDecode(&stage.coll, "DELETE", "arvados/v1/collections/"+stage.coll.UUID, nil, nil)
 103                 c.Check(err, check.IsNil)
 104         }
 105         if stage.proj.UUID != "" {
 106                 err := stage.arv.RequestAndDecode(&stage.proj, "DELETE", "arvados/v1/groups/"+stage.proj.UUID, nil, nil)
 107                 c.Check(err, check.IsNil)
 108         }
 109 }
 110
 111 func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
 112         stage := s.s3setup(c)
 113         defer stage.teardown(c)
 114
 115         bucket := stage.collbucket
 116         for _, trial := range []struct {
 117                 success   bool
 118                 signature int
 119                 accesskey string
 120                 secretkey string
 121         }{
 122                 {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
 123                 {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
 124                 {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
 125                 {false, aws.V2Signature, "none", "none"},
 126                 {false, aws.V2Signature, "none", arvadostest.ActiveToken},
 127
 128                 {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
 129                 {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
 130                 {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
 131                 {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
 132                 {false, aws.V4Signature, arvadostest.ActiveToken, ""},
 133                 {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
 134                 {false, aws.V4Signature, "none", arvadostest.ActiveToken},
 135                 {false, aws.V4Signature, "none", "none"},
 136         } {
 137                 c.Logf("%#v", trial)
 138                 bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
 139                 bucket.S3.Signature = trial.signature
 140                 _, err := bucket.GetReader("emptyfile")
 141                 if trial.success {
 142                         c.Check(err, check.IsNil)
 143                 } else {
 144                         c.Check(err, check.NotNil)
 145                 }
 146         }
 147 }
 148
 149 func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
 150         stage := s.s3setup(c)
 151         defer stage.teardown(c)
 152
 153         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 154                 c.Logf("bucket %s", bucket.Name)
 155                 exists, err := bucket.Exists("")
 156                 c.Check(err, check.IsNil)
 157                 c.Check(exists, check.Equals, true)
 158         }
 159 }
 160
 161 func (s *IntegrationSuite) TestS3CollectionGetObject(c *check.C) {
 162         stage := s.s3setup(c)
 163         defer stage.teardown(c)
 164         s.testS3GetObject(c, stage.collbucket, "")
 165 }
 166 func (s *IntegrationSuite) TestS3ProjectGetObject(c *check.C) {
 167         stage := s.s3setup(c)
 168         defer stage.teardown(c)
 169         s.testS3GetObject(c, stage.projbucket, stage.coll.Name+"/")
 170 }
 171 func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix string) {
 172         rdr, err := bucket.GetReader(prefix + "emptyfile")
 173         c.Assert(err, check.IsNil)
 174         buf, err := ioutil.ReadAll(rdr)
 175         c.Check(err, check.IsNil)
 176         c.Check(len(buf), check.Equals, 0)
 177         err = rdr.Close()
 178         c.Check(err, check.IsNil)
 179
 180         // GetObject
 181         rdr, err = bucket.GetReader(prefix + "missingfile")
 182         c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 183         c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 184         c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
 185
 186         // HeadObject
 187         exists, err := bucket.Exists(prefix + "missingfile")
 188         c.Check(err, check.IsNil)
 189         c.Check(exists, check.Equals, false)
 190
 191         // GetObject
 192         rdr, err = bucket.GetReader(prefix + "sailboat.txt")
 193         c.Assert(err, check.IsNil)
 194         buf, err = ioutil.ReadAll(rdr)
 195         c.Check(err, check.IsNil)
 196         c.Check(buf, check.DeepEquals, []byte("⛵\n"))
 197         err = rdr.Close()
 198         c.Check(err, check.IsNil)
 199
 200         // HeadObject
 201         resp, err := bucket.Head(prefix+"sailboat.txt", nil)
 202         c.Check(err, check.IsNil)
 203         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 204         c.Check(resp.ContentLength, check.Equals, int64(4))
 205 }
 206
 207 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
 208         stage := s.s3setup(c)
 209         defer stage.teardown(c)
 210         s.testS3PutObjectSuccess(c, stage.collbucket, "")
 211 }
 212 func (s *IntegrationSuite) TestS3ProjectPutObjectSuccess(c *check.C) {
 213         stage := s.s3setup(c)
 214         defer stage.teardown(c)
 215         s.testS3PutObjectSuccess(c, stage.projbucket, stage.coll.Name+"/")
 216 }
 217 func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, prefix string) {
 218         for _, trial := range []struct {
 219                 path        string
 220                 size        int
 221                 contentType string
 222         }{
 223                 {
 224                         path:        "newfile",
 225                         size:        128000000,
 226                         contentType: "application/octet-stream",
 227                 }, {
 228                         path:        "newdir/newfile",
 229                         size:        1 << 26,
 230                         contentType: "application/octet-stream",
 231                 }, {
 232                         path:        "newdir1/newdir2/newfile",
 233                         size:        0,
 234                         contentType: "application/octet-stream",
 235                 }, {
 236                         path:        "newdir1/newdir2/newdir3/",
 237                         size:        0,
 238                         contentType: "application/x-directory",
 239                 },
 240         } {
 241                 c.Logf("=== %v", trial)
 242
 243                 objname := prefix + trial.path
 244
 245                 _, err := bucket.GetReader(objname)
 246                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 247                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 248                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 249
 250                 buf := make([]byte, trial.size)
 251                 rand.Read(buf)
 252
 253                 err = bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 254                 c.Check(err, check.IsNil)
 255
 256                 rdr, err := bucket.GetReader(objname)
 257                 if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
 258                         c.Check(err, check.NotNil)
 259                         continue
 260                 } else if !c.Check(err, check.IsNil) {
 261                         continue
 262                 }
 263                 buf2, err := ioutil.ReadAll(rdr)
 264                 c.Check(err, check.IsNil)
 265                 c.Check(buf2, check.HasLen, len(buf))
 266                 c.Check(bytes.Equal(buf, buf2), check.Equals, true)
 267         }
 268 }
 269
 270 func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
 271         stage := s.s3setup(c)
 272         defer stage.teardown(c)
 273         bucket := stage.projbucket
 274
 275         for _, trial := range []struct {
 276                 path        string
 277                 size        int
 278                 contentType string
 279         }{
 280                 {
 281                         path:        "newfile",
 282                         size:        1234,
 283                         contentType: "application/octet-stream",
 284                 }, {
 285                         path:        "newdir/newfile",
 286                         size:        1234,
 287                         contentType: "application/octet-stream",
 288                 }, {
 289                         path:        "newdir2/",
 290                         size:        0,
 291                         contentType: "application/x-directory",
 292                 },
 293         } {
 294                 c.Logf("=== %v", trial)
 295
 296                 _, err := bucket.GetReader(trial.path)
 297                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 298                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 299                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 300
 301                 buf := make([]byte, trial.size)
 302                 rand.Read(buf)
 303
 304                 err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
 305                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
 306                 c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
 307                 c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
 308
 309                 _, err = bucket.GetReader(trial.path)
 310                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 311                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 312                 c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
 313         }
 314 }
 315
 316 func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) {
 317         stage := s.s3setup(c)
 318         defer stage.teardown(c)
 319         s.testS3DeleteObject(c, stage.collbucket, "")
 320 }
 321 func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) {
 322         stage := s.s3setup(c)
 323         defer stage.teardown(c)
 324         s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
 325 }
 326 func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
 327         s.testServer.Config.cluster.Collections.S3FolderObjects = true
 328         for _, trial := range []struct {
 329                 path string
 330         }{
 331                 {"/"},
 332                 {"nonexistentfile"},
 333                 {"emptyfile"},
 334                 {"sailboat.txt"},
 335                 {"sailboat.txt/"},
 336                 {"emptydir"},
 337                 {"emptydir/"},
 338         } {
 339                 objname := prefix + trial.path
 340                 comment := check.Commentf("objname %q", objname)
 341
 342                 err := bucket.Del(objname)
 343                 if trial.path == "/" {
 344                         c.Check(err, check.NotNil)
 345                         continue
 346                 }
 347                 c.Check(err, check.IsNil, comment)
 348                 _, err = bucket.GetReader(objname)
 349                 c.Check(err, check.NotNil, comment)
 350         }
 351 }
 352
 353 func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) {
 354         stage := s.s3setup(c)
 355         defer stage.teardown(c)
 356         s.testS3PutObjectFailure(c, stage.collbucket, "")
 357 }
 358 func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 359         stage := s.s3setup(c)
 360         defer stage.teardown(c)
 361         s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
 362 }
 363 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
 364         s.testServer.Config.cluster.Collections.S3FolderObjects = false
 365
 366         // Can't use V4 signature for these tests, because
 367         // double-slash is incorrectly cleaned by the aws.V4Signature,
 368         // resulting in a "bad signature" error. (Cleaning the path is
 369         // appropriate for other services, but not in S3 where object
 370         // names "foo//bar" and "foo/bar" are semantically different.)
 371         bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour)))
 372         bucket.S3.Signature = aws.V2Signature
 373
 374         var wg sync.WaitGroup
 375         for _, trial := range []struct {
 376                 path string
 377         }{
 378                 {
 379                         path: "emptyfile/newname", // emptyfile exists, see s3setup()
 380                 }, {
 381                         path: "emptyfile/", // emptyfile exists, see s3setup()
 382                 }, {
 383                         path: "emptydir", // dir already exists, see s3setup()
 384                 }, {
 385                         path: "emptydir/",
 386                 }, {
 387                         path: "emptydir//",
 388                 }, {
 389                         path: "newdir/",
 390                 }, {
 391                         path: "newdir//",
 392                 }, {
 393                         path: "/",
 394                 }, {
 395                         path: "//",
 396                 }, {
 397                         path: "foo//bar",
 398                 }, {
 399                         path: "",
 400                 },
 401         } {
 402                 trial := trial
 403                 wg.Add(1)
 404                 go func() {
 405                         defer wg.Done()
 406                         c.Logf("=== %v", trial)
 407
 408                         objname := prefix + trial.path
 409
 410                         buf := make([]byte, 1234)
 411                         rand.Read(buf)
 412
 413                         err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
 414                         if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
 415                                 return
 416                         }
 417
 418                         if objname != "" && objname != "/" {
 419                                 _, err = bucket.GetReader(objname)
 420                                 c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
 421                                 c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
 422                                 c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
 423                         }
 424                 }()
 425         }
 426         wg.Wait()
 427 }
 428
 429 func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
 430         fs, err := stage.coll.FileSystem(stage.arv, stage.kc)
 431         c.Assert(err, check.IsNil)
 432         for d := 0; d < dirs; d++ {
 433                 dir := fmt.Sprintf("dir%d", d)
 434                 c.Assert(fs.Mkdir(dir, 0755), check.IsNil)
 435                 for i := 0; i < filesPerDir; i++ {
 436                         f, err := fs.OpenFile(fmt.Sprintf("%s/file%d.txt", dir, i), os.O_CREATE|os.O_WRONLY, 0644)
 437                         c.Assert(err, check.IsNil)
 438                         c.Assert(f.Close(), check.IsNil)
 439                 }
 440         }
 441         c.Assert(fs.Sync(), check.IsNil)
 442 }
 443
 444 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
 445         stage := s.s3setup(c)
 446         defer stage.teardown(c)
 447         for _, trial := range []struct {
 448                 url            string
 449                 method         string
 450                 body           string
 451                 responseCode   int
 452                 responseRegexp []string
 453         }{
 454                 {
 455                         url:            "https://" + stage.collbucket.Name + ".example.com/",
 456                         method:         "GET",
 457                         responseCode:   http.StatusOK,
 458                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 459                 },
 460                 {
 461                         url:            "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
 462                         method:         "GET",
 463                         responseCode:   http.StatusOK,
 464                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 465                 },
 466                 {
 467                         url:            "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
 468                         method:         "GET",
 469                         responseCode:   http.StatusOK,
 470                         responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
 471                 },
 472                 {
 473                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
 474                         method:         "GET",
 475                         responseCode:   http.StatusOK,
 476                         responseRegexp: []string{`⛵\n`},
 477                 },
 478                 {
 479                         url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 480                         method:       "PUT",
 481                         body:         "boop",
 482                         responseCode: http.StatusOK,
 483                 },
 484                 {
 485                         url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
 486                         method:         "GET",
 487                         responseCode:   http.StatusOK,
 488                         responseRegexp: []string{`boop`},
 489                 },
 490         } {
 491                 url, err := url.Parse(trial.url)
 492                 c.Assert(err, check.IsNil)
 493                 req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
 494                 c.Assert(err, check.IsNil)
 495                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 496                 rr := httptest.NewRecorder()
 497                 s.testServer.Server.Handler.ServeHTTP(rr, req)
 498                 resp := rr.Result()
 499                 c.Check(resp.StatusCode, check.Equals, trial.responseCode)
 500                 body, err := ioutil.ReadAll(resp.Body)
 501                 c.Assert(err, check.IsNil)
 502                 for _, re := range trial.responseRegexp {
 503                         c.Check(string(body), check.Matches, re)
 504                 }
 505         }
 506 }
 507
 508 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
 509         stage := s.s3setup(c)
 510         defer stage.teardown(c)
 511         for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
 512                 req, err := http.NewRequest("GET", bucket.URL("/"), nil)
 513                 c.Check(err, check.IsNil)
 514                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 515                 req.URL.RawQuery = "versioning"
 516                 resp, err := http.DefaultClient.Do(req)
 517                 c.Assert(err, check.IsNil)
 518                 c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
 519                 buf, err := ioutil.ReadAll(resp.Body)
 520                 c.Assert(err, check.IsNil)
 521                 c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>\n")
 522         }
 523 }
 524
 525 // If there are no CommonPrefixes entries, the CommonPrefixes XML tag
 526 // should not appear at all.
 527 func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
 528         stage := s.s3setup(c)
 529         defer stage.teardown(c)
 530
 531         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 532         c.Assert(err, check.IsNil)
 533         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 534         req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
 535         resp, err := http.DefaultClient.Do(req)
 536         c.Assert(err, check.IsNil)
 537         buf, err := ioutil.ReadAll(resp.Body)
 538         c.Assert(err, check.IsNil)
 539         c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
 540 }
 541
 542 // If there is no delimiter in the request, or the results are not
 543 // truncated, the NextMarker XML tag should not appear in the response
 544 // body.
 545 func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) {
 546         stage := s.s3setup(c)
 547         defer stage.teardown(c)
 548
 549         for _, query := range []string{"prefix=e&delimiter=/", ""} {
 550                 req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 551                 c.Assert(err, check.IsNil)
 552                 req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 553                 req.URL.RawQuery = query
 554                 resp, err := http.DefaultClient.Do(req)
 555                 c.Assert(err, check.IsNil)
 556                 buf, err := ioutil.ReadAll(resp.Body)
 557                 c.Assert(err, check.IsNil)
 558                 c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`)
 559         }
 560 }
 561
 562 // List response should include KeyCount field.
 563 func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) {
 564         stage := s.s3setup(c)
 565         defer stage.teardown(c)
 566
 567         req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
 568         c.Assert(err, check.IsNil)
 569         req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
 570         req.URL.RawQuery = "prefix=&delimiter=/"
 571         resp, err := http.DefaultClient.Do(req)
 572         c.Assert(err, check.IsNil)
 573         buf, err := ioutil.ReadAll(resp.Body)
 574         c.Assert(err, check.IsNil)
 575         c.Check(string(buf), check.Matches, `(?ms).*<KeyCount>2</KeyCount>.*`)
 576 }
 577
 578 func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
 579         stage := s.s3setup(c)
 580         defer stage.teardown(c)
 581
 582         var markers int
 583         for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
 584                 dirs := 2
 585                 filesPerDir := 1001
 586                 stage.writeBigDirs(c, dirs, filesPerDir)
 587                 // Total # objects is:
 588                 //                 2 file entries from s3setup (emptyfile and sailboat.txt)
 589                 //                +1 fake "directory" marker from s3setup (emptydir) (if enabled)
 590                 //             +dirs fake "directory" marker from writeBigDirs (dir0/, dir1/) (if enabled)
 591                 // +filesPerDir*dirs file entries from writeBigDirs (dir0/file0.txt, etc.)
 592                 s.testS3List(c, stage.collbucket, "", 4000, markers+2+(filesPerDir+markers)*dirs)
 593                 s.testS3List(c, stage.collbucket, "", 131, markers+2+(filesPerDir+markers)*dirs)
 594                 s.testS3List(c, stage.collbucket, "dir0/", 71, filesPerDir+markers)
 595         }
 596 }
 597 func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
 598         c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
 599         expectPageSize := pageSize
 600         if expectPageSize > 1000 {
 601                 expectPageSize = 1000
 602         }
 603         gotKeys := map[string]s3.Key{}
 604         nextMarker := ""
 605         pages := 0
 606         for {
 607                 resp, err := bucket.List(prefix, "", nextMarker, pageSize)
 608                 if !c.Check(err, check.IsNil) {
 609                         break
 610                 }
 611                 c.Check(len(resp.Contents) <= expectPageSize, check.Equals, true)
 612                 if pages++; !c.Check(pages <= (expectFiles/expectPageSize)+1, check.Equals, true) {
 613                         break
 614                 }
 615                 for _, key := range resp.Contents {
 616                         gotKeys[key.Key] = key
 617                         if strings.Contains(key.Key, "sailboat.txt") {
 618                                 c.Check(key.Size, check.Equals, int64(4))
 619                         }
 620                 }
 621                 if !resp.IsTruncated {
 622                         c.Check(resp.NextMarker, check.Equals, "")
 623                         break
 624                 }
 625                 if !c.Check(resp.NextMarker, check.Not(check.Equals), "") {
 626                         break
 627                 }
 628                 nextMarker = resp.NextMarker
 629         }
 630         c.Check(len(gotKeys), check.Equals, expectFiles)
 631 }
 632
 633 func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
 634         for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
 635                 s.testS3CollectionListRollup(c)
 636         }
 637 }
 638
 639 func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) {
 640         stage := s.s3setup(c)
 641         defer stage.teardown(c)
 642
 643         dirs := 2
 644         filesPerDir := 500
 645         stage.writeBigDirs(c, dirs, filesPerDir)
 646         err := stage.collbucket.PutReader("dingbats", &bytes.Buffer{}, 0, "application/octet-stream", s3.Private, s3.Options{})
 647         c.Assert(err, check.IsNil)
 648         var allfiles []string
 649         for marker := ""; ; {
 650                 resp, err := stage.collbucket.List("", "", marker, 20000)
 651                 c.Check(err, check.IsNil)
 652                 for _, key := range resp.Contents {
 653                         if len(allfiles) == 0 || allfiles[len(allfiles)-1] != key.Key {
 654                                 allfiles = append(allfiles, key.Key)
 655                         }
 656                 }
 657                 marker = resp.NextMarker
 658                 if marker == "" {
 659                         break
 660                 }
 661         }
 662         markers := 0
 663         if s.testServer.Config.cluster.Collections.S3FolderObjects {
 664                 markers = 1
 665         }
 666         c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
 667
 668         gotDirMarker := map[string]bool{}
 669         for _, name := range allfiles {
 670                 isDirMarker := strings.HasSuffix(name, "/")
 671                 if markers == 0 {
 672                         c.Check(isDirMarker, check.Equals, false, check.Commentf("name %q", name))
 673                 } else if isDirMarker {
 674                         gotDirMarker[name] = true
 675                 } else if i := strings.LastIndex(name, "/"); i >= 0 {
 676                         c.Check(gotDirMarker[name[:i+1]], check.Equals, true, check.Commentf("name %q", name))
 677                         gotDirMarker[name[:i+1]] = true // skip redundant complaints about this dir marker
 678                 }
 679         }
 680
 681         for _, trial := range []struct {
 682                 prefix    string
 683                 delimiter string
 684                 marker    string
 685         }{
 686                 {"", "", ""},
 687                 {"di", "/", ""},
 688                 {"di", "r", ""},
 689                 {"di", "n", ""},
 690                 {"dir0", "/", ""},
 691                 {"dir0/", "/", ""},
 692                 {"dir0/f", "/", ""},
 693                 {"dir0", "", ""},
 694                 {"dir0/", "", ""},
 695                 {"dir0/f", "", ""},
 696                 {"dir0", "/", "dir0/file14.txt"},       // no commonprefixes
 697                 {"", "", "dir0/file14.txt"},            // middle page, skip walking dir1
 698                 {"", "", "dir1/file14.txt"},            // middle page, skip walking dir0
 699                 {"", "", "dir1/file498.txt"},           // last page of results
 700                 {"dir1/file", "", "dir1/file498.txt"},  // last page of results, with prefix
 701                 {"dir1/file", "/", "dir1/file498.txt"}, // last page of results, with prefix + delimiter
 702                 {"dir1", "Z", "dir1/file498.txt"},      // delimiter "Z" never appears
 703                 {"dir2", "/", ""},                      // prefix "dir2" does not exist
 704                 {"", "/", ""},
 705         } {
 706                 c.Logf("\n\n=== trial %+v markers=%d", trial, markers)
 707
 708                 maxKeys := 20
 709                 resp, err := stage.collbucket.List(trial.prefix, trial.delimiter, trial.marker, maxKeys)
 710                 c.Check(err, check.IsNil)
 711                 if resp.IsTruncated && trial.delimiter == "" {
 712                         // goamz List method fills in the missing
 713                         // NextMarker field if resp.IsTruncated, so
 714                         // now we can't really tell whether it was
 715                         // sent by the server or by goamz. In cases
 716                         // where it should be empty but isn't, assume
 717                         // it's goamz's fault.
 718                         resp.NextMarker = ""
 719                 }
 720
 721                 var expectKeys []string
 722                 var expectPrefixes []string
 723                 var expectNextMarker string
 724                 var expectTruncated bool
 725                 for _, key := range allfiles {
 726                         full := len(expectKeys)+len(expectPrefixes) >= maxKeys
 727                         if !strings.HasPrefix(key, trial.prefix) || key < trial.marker {
 728                                 continue
 729                         } else if idx := strings.Index(key[len(trial.prefix):], trial.delimiter); trial.delimiter != "" && idx >= 0 {
 730                                 prefix := key[:len(trial.prefix)+idx+1]
 731                                 if len(expectPrefixes) > 0 && expectPrefixes[len(expectPrefixes)-1] == prefix {
 732                                         // same prefix as previous key
 733                                 } else if full {
 734                                         expectNextMarker = key
 735                                         expectTruncated = true
 736                                 } else {
 737                                         expectPrefixes = append(expectPrefixes, prefix)
 738                                 }
 739                         } else if full {
 740                                 if trial.delimiter != "" {
 741                                         expectNextMarker = key
 742                                 }
 743                                 expectTruncated = true
 744                                 break
 745                         } else {
 746                                 expectKeys = append(expectKeys, key)
 747                         }
 748                 }
 749
 750                 var gotKeys []string
 751                 for _, key := range resp.Contents {
 752                         gotKeys = append(gotKeys, key.Key)
 753                 }
 754                 var gotPrefixes []string
 755                 for _, prefix := range resp.CommonPrefixes {
 756                         gotPrefixes = append(gotPrefixes, prefix)
 757                 }
 758                 commentf := check.Commentf("trial %+v markers=%d", trial, markers)
 759                 c.Check(gotKeys, check.DeepEquals, expectKeys, commentf)
 760                 c.Check(gotPrefixes, check.DeepEquals, expectPrefixes, commentf)
 761                 c.Check(resp.NextMarker, check.Equals, expectNextMarker, commentf)
 762                 c.Check(resp.IsTruncated, check.Equals, expectTruncated, commentf)
 763                 c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
 764         }
 765 }
 766
 767 // TestS3cmd checks compatibility with the s3cmd command line tool, if
 768 // it's installed. As of Debian buster, s3cmd is only in backports, so
 769 // `arvados-server install` don't install it, and this test skips if
 770 // it's not installed.
 771 func (s *IntegrationSuite) TestS3cmd(c *check.C) {
 772         if _, err := exec.LookPath("s3cmd"); err != nil {
 773                 c.Skip("s3cmd not found")
 774                 return
 775         }
 776
 777         stage := s.s3setup(c)
 778         defer stage.teardown(c)
 779
 780         cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
 781         buf, err := cmd.CombinedOutput()
 782         c.Check(err, check.IsNil)
 783         c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
 784 }
 785
 786 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
 787         stage := s.s3setup(c)
 788         defer stage.teardown(c)
 789
 790         hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
 791         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 792         c.Check(body, check.Equals, "⛵\n")
 793 }