services/keep-web/cadaver_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "net/url"
  13         "os"
  14         "os/exec"
  15         "path/filepath"
  16         "strings"
  17         "time"
  18
  19         "git.curoverse.com/arvados.git/sdk/go/arvados"
  20         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  21         check "gopkg.in/check.v1"
  22 )
  23
  24 func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
  25         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  26                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
  27                 w := "/c=" + newCollection.UUID + "/"
  28                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  29                 return r, w, pdh
  30         }, nil)
  31 }
  32
  33 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
  34         s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
  35                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
  36                 w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
  37                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
  38                 return r, w, pdh
  39         }, nil)
  40 }
  41
  42 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
  43         rpath := "/users/active/foo_file_in_dir/"
  44         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  45                 wpath := "/users/active/" + newCollection.Name
  46                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  47                 return rpath, wpath, pdh
  48         }, func(path string) bool {
  49                 // Skip tests that rely on writes, because /users/
  50                 // tree is read-only.
  51                 return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
  52         })
  53 }
  54
  55 func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
  56         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
  57
  58         testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
  59
  60         tempdir, err := ioutil.TempDir("", "keep-web-test-")
  61         c.Assert(err, check.IsNil)
  62         defer os.RemoveAll(tempdir)
  63
  64         localfile, err := ioutil.TempFile(tempdir, "localfile")
  65         c.Assert(err, check.IsNil)
  66         localfile.Write(testdata)
  67
  68         emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
  69         c.Assert(err, check.IsNil)
  70
  71         checkfile, err := ioutil.TempFile(tempdir, "checkfile")
  72         c.Assert(err, check.IsNil)
  73
  74         var newCollection arvados.Collection
  75         arv := arvados.NewClientFromEnv()
  76         arv.AuthToken = arvadostest.ActiveToken
  77         err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", bytes.NewBufferString(url.Values{"collection": {"{}"}}.Encode()), nil)
  78         c.Assert(err, check.IsNil)
  79
  80         readPath, writePath, pdhPath := pathFunc(newCollection)
  81
  82         matchToday := time.Now().Format("Jan +2")
  83
  84         type testcase struct {
  85                 path  string
  86                 cmd   string
  87                 match string
  88                 data  []byte
  89         }
  90         for _, trial := range []testcase{
  91                 {
  92                         path:  readPath,
  93                         cmd:   "ls\n",
  94                         match: `(?ms).*dir1 *0 .*`,
  95                 },
  96                 {
  97                         path:  readPath,
  98                         cmd:   "ls dir1\n",
  99                         match: `(?ms).*bar *3.*foo *3 .*`,
 100                 },
 101                 {
 102                         path:  readPath + "_/dir1",
 103                         cmd:   "ls\n",
 104                         match: `(?ms).*bar *3.*foo *3 .*`,
 105                 },
 106                 {
 107                         path:  readPath + "dir1/",
 108                         cmd:   "ls\n",
 109                         match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
 110                 },
 111                 {
 112                         path:  writePath,
 113                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 114                         match: `(?ms).*Not Found.*`,
 115                 },
 116                 {
 117                         path:  writePath,
 118                         cmd:   "put '" + emptyfile.Name() + "' emptyfile\n",
 119                         match: `(?ms).*Uploading .* succeeded.*`,
 120                 },
 121                 {
 122                         path:  writePath,
 123                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 124                         match: `(?ms).*Downloading .* succeeded.*`,
 125                         data:  []byte{},
 126                 },
 127                 {
 128                         path:  writePath,
 129                         cmd:   "put '" + localfile.Name() + "' testfile\n",
 130                         match: `(?ms).*Uploading .* succeeded.*`,
 131                 },
 132                 {
 133                         path:  writePath,
 134                         cmd:   "get testfile '" + checkfile.Name() + "'\n",
 135                         match: `(?ms).*succeeded.*`,
 136                         data:  testdata,
 137                 },
 138                 {
 139                         path:  writePath,
 140                         cmd:   "move testfile newdir0/\n",
 141                         match: `(?ms).*Moving .* succeeded.*`,
 142                 },
 143                 {
 144                         path:  writePath,
 145                         cmd:   "move testfile newdir0/\n",
 146                         match: `(?ms).*Moving .* failed.*`,
 147                 },
 148                 {
 149                         path:  writePath,
 150                         cmd:   "lock newdir0/testfile\n",
 151                         match: `(?ms).*Locking .* succeeded.*`,
 152                 },
 153                 {
 154                         path:  writePath,
 155                         cmd:   "unlock newdir0/testfile\nasdf\n",
 156                         match: `(?ms).*Unlocking .* succeeded.*`,
 157                 },
 158                 {
 159                         path:  writePath,
 160                         cmd:   "ls\n",
 161                         match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
 162                 },
 163                 {
 164                         path:  writePath,
 165                         cmd:   "move newdir0/testfile emptyfile/bogus/\n",
 166                         match: `(?ms).*Moving .* failed.*`,
 167                 },
 168                 {
 169                         path:  writePath,
 170                         cmd:   "mkcol newdir1\n",
 171                         match: `(?ms).*Creating .* succeeded.*`,
 172                 },
 173                 {
 174                         path:  writePath,
 175                         cmd:   "move newdir1/ newdir1x/\n",
 176                         match: `(?ms).*Moving .* succeeded.*`,
 177                 },
 178                 {
 179                         path:  writePath,
 180                         cmd:   "move newdir1x newdir1\n",
 181                         match: `(?ms).*Moving .* succeeded.*`,
 182                 },
 183                 {
 184                         path:  writePath,
 185                         cmd:   "move newdir0/testfile newdir1/\n",
 186                         match: `(?ms).*Moving .* succeeded.*`,
 187                 },
 188                 {
 189                         path:  writePath,
 190                         cmd:   "move newdir1 newdir1/\n",
 191                         match: `(?ms).*Moving .* failed.*`,
 192                 },
 193                 {
 194                         path:  writePath,
 195                         cmd:   "get newdir1/testfile '" + checkfile.Name() + "'\n",
 196                         match: `(?ms).*succeeded.*`,
 197                         data:  testdata,
 198                 },
 199                 {
 200                         path:  writePath,
 201                         cmd:   "put '" + localfile.Name() + "' newdir1/testfile1\n",
 202                         match: `(?ms).*Uploading .* succeeded.*`,
 203                 },
 204                 {
 205                         path:  writePath,
 206                         cmd:   "mkcol newdir2\n",
 207                         match: `(?ms).*Creating .* succeeded.*`,
 208                 },
 209                 {
 210                         path:  writePath,
 211                         cmd:   "put '" + localfile.Name() + "' newdir2/testfile2\n",
 212                         match: `(?ms).*Uploading .* succeeded.*`,
 213                 },
 214                 {
 215                         path:  writePath,
 216                         cmd:   "copy newdir2/testfile2 testfile3\n",
 217                         match: `(?ms).*succeeded.*`,
 218                 },
 219                 {
 220                         path:  writePath,
 221                         cmd:   "get testfile3 '" + checkfile.Name() + "'\n",
 222                         match: `(?ms).*succeeded.*`,
 223                         data:  testdata,
 224                 },
 225                 {
 226                         path:  writePath,
 227                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 228                         match: `(?ms).*succeeded.*`,
 229                         data:  testdata,
 230                 },
 231                 {
 232                         path:  writePath,
 233                         cmd:   "rmcol newdir2\n",
 234                         match: `(?ms).*Deleting collection .* succeeded.*`,
 235                 },
 236                 {
 237                         path:  writePath,
 238                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 239                         match: `(?ms).*Downloading .* failed.*`,
 240                 },
 241                 {
 242                         path:  "/c=" + arvadostest.UserAgreementCollection + "/t=" + arv.AuthToken + "/",
 243                         cmd:   "put '" + localfile.Name() + "' foo\n",
 244                         match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
 245                 },
 246                 {
 247                         path:  pdhPath,
 248                         cmd:   "put '" + localfile.Name() + "' foo\n",
 249                         match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
 250                 },
 251                 {
 252                         path:  pdhPath,
 253                         cmd:   "move foo bar\n",
 254                         match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
 255                 },
 256                 {
 257                         path:  pdhPath,
 258                         cmd:   "copy foo bar\n",
 259                         match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
 260                 },
 261                 {
 262                         path:  pdhPath,
 263                         cmd:   "delete foo\n",
 264                         match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
 265                 },
 266                 {
 267                         path:  pdhPath,
 268                         cmd:   "lock foo\n",
 269                         match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
 270                 },
 271         } {
 272                 c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
 273                 if skip != nil && skip(trial.path) {
 274                         c.Log("(skip)")
 275                         continue
 276                 }
 277
 278                 os.Remove(checkfile.Name())
 279
 280                 stdout := s.runCadaver(c, password, trial.path, trial.cmd)
 281                 c.Check(stdout, check.Matches, trial.match)
 282
 283                 if trial.data == nil {
 284                         continue
 285                 }
 286                 checkfile, err = os.Open(checkfile.Name())
 287                 c.Assert(err, check.IsNil)
 288                 checkfile.Seek(0, os.SEEK_SET)
 289                 got, err := ioutil.ReadAll(checkfile)
 290                 c.Check(got, check.DeepEquals, trial.data)
 291                 c.Check(err, check.IsNil)
 292         }
 293 }
 294
 295 func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
 296         for _, path := range []string{"/by_id", "/by_id/"} {
 297                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 298                 c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
 299         }
 300         for _, path := range []string{
 301                 "/by_id/" + arvadostest.FooCollectionPDH,
 302                 "/by_id/" + arvadostest.FooCollectionPDH + "/",
 303                 "/by_id/" + arvadostest.FooCollection,
 304                 "/by_id/" + arvadostest.FooCollection + "/",
 305         } {
 306                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 307                 c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
 308         }
 309 }
 310
 311 func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
 312         for _, path := range []string{"/"} {
 313                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 314                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
 315                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
 316         }
 317         for _, path := range []string{"/users", "/users/"} {
 318                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 319                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
 320         }
 321         for _, path := range []string{"/users/active", "/users/active/"} {
 322                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 323                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
 324                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
 325         }
 326         for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
 327                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 328                 c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
 329         }
 330 }
 331
 332 func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
 333         tempdir, err := ioutil.TempDir("", "keep-web-test-")
 334         c.Assert(err, check.IsNil)
 335         defer os.RemoveAll(tempdir)
 336
 337         cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
 338         if password != "" {
 339                 // cadaver won't try username/password authentication
 340                 // unless the server responds 401 to an
 341                 // unauthenticated request, which it only does in
 342                 // AttachmentOnlyHost, TrustAllContent, and
 343                 // per-collection vhost cases.
 344                 s.testServer.Config.AttachmentOnlyHost = s.testServer.Addr
 345
 346                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 347                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 348                 c.Assert(err, check.IsNil)
 349                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 350                 c.Assert(err, check.IsNil)
 351                 c.Assert(f.Close(), check.IsNil)
 352         }
 353         cmd.Stdin = bytes.NewBufferString(stdin)
 354         stdout, err := cmd.StdoutPipe()
 355         c.Assert(err, check.Equals, nil)
 356         cmd.Stderr = cmd.Stdout
 357         go cmd.Start()
 358
 359         var buf bytes.Buffer
 360         _, err = io.Copy(&buf, stdout)
 361         c.Check(err, check.Equals, nil)
 362         err = cmd.Wait()
 363         c.Check(err, check.Equals, nil)
 364         return buf.String()
 365 }