services/keep-web/cadaver_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "strings"
  16         "time"
  17
  18         "git.curoverse.com/arvados.git/sdk/go/arvados"
  19         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
  24         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  25                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
  26                 w := "/c=" + newCollection.UUID + "/"
  27                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  28                 return r, w, pdh
  29         }, nil)
  30 }
  31
  32 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
  33         s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
  34                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
  35                 w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
  36                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
  37                 return r, w, pdh
  38         }, nil)
  39 }
  40
  41 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
  42         rpath := "/users/active/foo_file_in_dir/"
  43         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  44                 wpath := "/users/active/" + newCollection.Name
  45                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  46                 return rpath, wpath, pdh
  47         }, func(path string) bool {
  48                 // Skip tests that rely on writes, because /users/
  49                 // tree is read-only.
  50                 return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
  51         })
  52 }
  53
  54 func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
  55         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
  56
  57         testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
  58
  59         tempdir, err := ioutil.TempDir("", "keep-web-test-")
  60         c.Assert(err, check.IsNil)
  61         defer os.RemoveAll(tempdir)
  62
  63         localfile, err := ioutil.TempFile(tempdir, "localfile")
  64         c.Assert(err, check.IsNil)
  65         localfile.Write(testdata)
  66
  67         emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
  68         c.Assert(err, check.IsNil)
  69
  70         checkfile, err := ioutil.TempFile(tempdir, "checkfile")
  71         c.Assert(err, check.IsNil)
  72
  73         var newCollection arvados.Collection
  74         arv := arvados.NewClientFromEnv()
  75         arv.AuthToken = arvadostest.ActiveToken
  76         err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{}})
  77         c.Assert(err, check.IsNil)
  78
  79         readPath, writePath, pdhPath := pathFunc(newCollection)
  80
  81         matchToday := time.Now().Format("Jan +2")
  82
  83         type testcase struct {
  84                 path  string
  85                 cmd   string
  86                 match string
  87                 data  []byte
  88         }
  89         for _, trial := range []testcase{
  90                 {
  91                         path:  readPath,
  92                         cmd:   "ls\n",
  93                         match: `(?ms).*dir1 *0 .*`,
  94                 },
  95                 {
  96                         path:  readPath,
  97                         cmd:   "ls dir1\n",
  98                         match: `(?ms).*bar *3.*foo *3 .*`,
  99                 },
 100                 {
 101                         path:  readPath + "_/dir1",
 102                         cmd:   "ls\n",
 103                         match: `(?ms).*bar *3.*foo *3 .*`,
 104                 },
 105                 {
 106                         path:  readPath + "dir1/",
 107                         cmd:   "ls\n",
 108                         match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
 109                 },
 110                 {
 111                         path:  writePath,
 112                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 113                         match: `(?ms).*Not Found.*`,
 114                 },
 115                 {
 116                         path:  writePath,
 117                         cmd:   "put '" + emptyfile.Name() + "' emptyfile\n",
 118                         match: `(?ms).*Uploading .* succeeded.*`,
 119                 },
 120                 {
 121                         path:  writePath,
 122                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 123                         match: `(?ms).*Downloading .* succeeded.*`,
 124                         data:  []byte{},
 125                 },
 126                 {
 127                         path:  writePath,
 128                         cmd:   "put '" + localfile.Name() + "' testfile\n",
 129                         match: `(?ms).*Uploading .* succeeded.*`,
 130                 },
 131                 {
 132                         path:  writePath,
 133                         cmd:   "get testfile '" + checkfile.Name() + "'\n",
 134                         match: `(?ms).*succeeded.*`,
 135                         data:  testdata,
 136                 },
 137                 {
 138                         path:  writePath,
 139                         cmd:   "move testfile newdir0/\n",
 140                         match: `(?ms).*Moving .* succeeded.*`,
 141                 },
 142                 {
 143                         path:  writePath,
 144                         cmd:   "move testfile newdir0/\n",
 145                         match: `(?ms).*Moving .* failed.*`,
 146                 },
 147                 {
 148                         path:  writePath,
 149                         cmd:   "lock newdir0/testfile\n",
 150                         match: `(?ms).*Locking .* succeeded.*`,
 151                 },
 152                 {
 153                         path:  writePath,
 154                         cmd:   "unlock newdir0/testfile\nasdf\n",
 155                         match: `(?ms).*Unlocking .* succeeded.*`,
 156                 },
 157                 {
 158                         path:  writePath,
 159                         cmd:   "ls\n",
 160                         match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
 161                 },
 162                 {
 163                         path:  writePath,
 164                         cmd:   "move newdir0/testfile emptyfile/bogus/\n",
 165                         match: `(?ms).*Moving .* failed.*`,
 166                 },
 167                 {
 168                         path:  writePath,
 169                         cmd:   "mkcol newdir1\n",
 170                         match: `(?ms).*Creating .* succeeded.*`,
 171                 },
 172                 {
 173                         path:  writePath,
 174                         cmd:   "move newdir1/ newdir1x/\n",
 175                         match: `(?ms).*Moving .* succeeded.*`,
 176                 },
 177                 {
 178                         path:  writePath,
 179                         cmd:   "move newdir1x newdir1\n",
 180                         match: `(?ms).*Moving .* succeeded.*`,
 181                 },
 182                 {
 183                         path:  writePath,
 184                         cmd:   "move newdir0/testfile newdir1/\n",
 185                         match: `(?ms).*Moving .* succeeded.*`,
 186                 },
 187                 {
 188                         path:  writePath,
 189                         cmd:   "move newdir1 newdir1/\n",
 190                         match: `(?ms).*Moving .* failed.*`,
 191                 },
 192                 {
 193                         path:  writePath,
 194                         cmd:   "get newdir1/testfile '" + checkfile.Name() + "'\n",
 195                         match: `(?ms).*succeeded.*`,
 196                         data:  testdata,
 197                 },
 198                 {
 199                         path:  writePath,
 200                         cmd:   "put '" + localfile.Name() + "' newdir1/testfile1\n",
 201                         match: `(?ms).*Uploading .* succeeded.*`,
 202                 },
 203                 {
 204                         path:  writePath,
 205                         cmd:   "mkcol newdir2\n",
 206                         match: `(?ms).*Creating .* succeeded.*`,
 207                 },
 208                 {
 209                         path:  writePath,
 210                         cmd:   "put '" + localfile.Name() + "' newdir2/testfile2\n",
 211                         match: `(?ms).*Uploading .* succeeded.*`,
 212                 },
 213                 {
 214                         path:  writePath,
 215                         cmd:   "copy newdir2/testfile2 testfile3\n",
 216                         match: `(?ms).*succeeded.*`,
 217                 },
 218                 {
 219                         path:  writePath,
 220                         cmd:   "get testfile3 '" + checkfile.Name() + "'\n",
 221                         match: `(?ms).*succeeded.*`,
 222                         data:  testdata,
 223                 },
 224                 {
 225                         path:  writePath,
 226                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 227                         match: `(?ms).*succeeded.*`,
 228                         data:  testdata,
 229                 },
 230                 {
 231                         path:  writePath,
 232                         cmd:   "rmcol newdir2\n",
 233                         match: `(?ms).*Deleting collection .* succeeded.*`,
 234                 },
 235                 {
 236                         path:  writePath,
 237                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 238                         match: `(?ms).*Downloading .* failed.*`,
 239                 },
 240                 {
 241                         path:  "/c=" + arvadostest.UserAgreementCollection + "/t=" + arv.AuthToken + "/",
 242                         cmd:   "put '" + localfile.Name() + "' foo\n",
 243                         match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
 244                 },
 245                 {
 246                         path:  pdhPath,
 247                         cmd:   "put '" + localfile.Name() + "' foo\n",
 248                         match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
 249                 },
 250                 {
 251                         path:  pdhPath,
 252                         cmd:   "move foo bar\n",
 253                         match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
 254                 },
 255                 {
 256                         path:  pdhPath,
 257                         cmd:   "copy foo bar\n",
 258                         match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
 259                 },
 260                 {
 261                         path:  pdhPath,
 262                         cmd:   "delete foo\n",
 263                         match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
 264                 },
 265                 {
 266                         path:  pdhPath,
 267                         cmd:   "lock foo\n",
 268                         match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
 269                 },
 270         } {
 271                 c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
 272                 if skip != nil && skip(trial.path) {
 273                         c.Log("(skip)")
 274                         continue
 275                 }
 276
 277                 os.Remove(checkfile.Name())
 278
 279                 stdout := s.runCadaver(c, password, trial.path, trial.cmd)
 280                 c.Check(stdout, check.Matches, trial.match)
 281
 282                 if trial.data == nil {
 283                         continue
 284                 }
 285                 checkfile, err = os.Open(checkfile.Name())
 286                 c.Assert(err, check.IsNil)
 287                 checkfile.Seek(0, os.SEEK_SET)
 288                 got, err := ioutil.ReadAll(checkfile)
 289                 c.Check(got, check.DeepEquals, trial.data)
 290                 c.Check(err, check.IsNil)
 291         }
 292 }
 293
 294 func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
 295         for _, path := range []string{"/by_id", "/by_id/"} {
 296                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 297                 c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
 298         }
 299         for _, path := range []string{
 300                 "/by_id/" + arvadostest.FooCollectionPDH,
 301                 "/by_id/" + arvadostest.FooCollectionPDH + "/",
 302                 "/by_id/" + arvadostest.FooCollection,
 303                 "/by_id/" + arvadostest.FooCollection + "/",
 304         } {
 305                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 306                 c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
 307         }
 308 }
 309
 310 func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
 311         for _, path := range []string{"/"} {
 312                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 313                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
 314                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
 315         }
 316         for _, path := range []string{"/users", "/users/"} {
 317                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 318                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
 319         }
 320         for _, path := range []string{"/users/active", "/users/active/"} {
 321                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 322                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
 323                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
 324         }
 325         for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
 326                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 327                 c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
 328         }
 329 }
 330
 331 func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
 332         tempdir, err := ioutil.TempDir("", "keep-web-test-")
 333         c.Assert(err, check.IsNil)
 334         defer os.RemoveAll(tempdir)
 335
 336         cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
 337         if password != "" {
 338                 // cadaver won't try username/password authentication
 339                 // unless the server responds 401 to an
 340                 // unauthenticated request, which it only does in
 341                 // AttachmentOnlyHost, TrustAllContent, and
 342                 // per-collection vhost cases.
 343                 s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
 344
 345                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 346                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 347                 c.Assert(err, check.IsNil)
 348                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 349                 c.Assert(err, check.IsNil)
 350                 c.Assert(f.Close(), check.IsNil)
 351         }
 352         cmd.Stdin = bytes.NewBufferString(stdin)
 353         stdout, err := cmd.StdoutPipe()
 354         c.Assert(err, check.Equals, nil)
 355         cmd.Stderr = cmd.Stdout
 356         go cmd.Start()
 357
 358         var buf bytes.Buffer
 359         _, err = io.Copy(&buf, stdout)
 360         c.Check(err, check.Equals, nil)
 361         err = cmd.Wait()
 362         c.Check(err, check.Equals, nil)
 363         return buf.String()
 364 }