services/keep-web/cadaver_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "strings"
  16         "time"
  17
  18         "git.arvados.org/arvados.git/sdk/go/arvados"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
  24         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  25                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
  26                 w := "/c=" + newCollection.UUID + "/"
  27                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  28                 return r, w, pdh
  29         }, nil)
  30 }
  31
  32 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
  33         s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
  34                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
  35                 w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
  36                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
  37                 return r, w, pdh
  38         }, nil)
  39 }
  40
  41 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
  42         rpath := "/users/active/foo_file_in_dir/"
  43         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  44                 wpath := "/users/active/" + newCollection.Name
  45                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  46                 return rpath, wpath, pdh
  47         }, func(path string) bool {
  48                 // Skip tests that rely on writes, because /users/
  49                 // tree is read-only.
  50                 return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
  51         })
  52 }
  53
  54 func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
  55         testdata := "the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand"
  56
  57         tempdir, err := ioutil.TempDir("", "keep-web-test-")
  58         c.Assert(err, check.IsNil)
  59         defer os.RemoveAll(tempdir)
  60
  61         localfile, err := ioutil.TempFile(tempdir, "localfile")
  62         c.Assert(err, check.IsNil)
  63         localfile.Write([]byte(testdata))
  64
  65         emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
  66         c.Assert(err, check.IsNil)
  67
  68         checkfile, err := ioutil.TempFile(tempdir, "checkfile")
  69         c.Assert(err, check.IsNil)
  70
  71         var newCollection arvados.Collection
  72         arv := arvados.NewClientFromEnv()
  73         arv.AuthToken = arvadostest.ActiveToken
  74         err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{}})
  75         c.Assert(err, check.IsNil)
  76
  77         readPath, writePath, pdhPath := pathFunc(newCollection)
  78
  79         matchToday := time.Now().Format("Jan +2")
  80
  81         type testcase struct {
  82                 path           string
  83                 cmd            string
  84                 match          string
  85                 data           string
  86                 checkemptydata bool
  87         }
  88         for _, trial := range []testcase{
  89                 {
  90                         path:  readPath,
  91                         cmd:   "ls\n",
  92                         match: `(?ms).*dir1 *0 .*`,
  93                 },
  94                 {
  95                         path:  readPath,
  96                         cmd:   "ls dir1\n",
  97                         match: `(?ms).*bar *3.*foo *3 .*`,
  98                 },
  99                 {
 100                         path:  readPath + "_/dir1",
 101                         cmd:   "ls\n",
 102                         match: `(?ms).*bar *3.*foo *3 .*`,
 103                 },
 104                 {
 105                         path:  readPath + "dir1/",
 106                         cmd:   "ls\n",
 107                         match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
 108                 },
 109                 {
 110                         path:  writePath,
 111                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 112                         match: `(?ms).*Not Found.*`,
 113                 },
 114                 {
 115                         path:  writePath,
 116                         cmd:   "put '" + emptyfile.Name() + "' emptyfile\n",
 117                         match: `(?ms).*Uploading .* succeeded.*`,
 118                 },
 119                 {
 120                         path:           writePath,
 121                         cmd:            "get emptyfile '" + checkfile.Name() + "'\n",
 122                         match:          `(?ms).*Downloading .* succeeded.*`,
 123                         checkemptydata: true,
 124                 },
 125                 {
 126                         path:  writePath,
 127                         cmd:   "put '" + localfile.Name() + "' testfile\n",
 128                         match: `(?ms).*Uploading .* succeeded.*`,
 129                 },
 130                 {
 131                         path:  writePath,
 132                         cmd:   "get testfile '" + checkfile.Name() + "'\n",
 133                         match: `(?ms).*succeeded.*`,
 134                         data:  testdata,
 135                 },
 136                 {
 137                         path:  writePath,
 138                         cmd:   "move testfile \"test &#!%20 file\"\n",
 139                         match: `(?ms).*Moving .* succeeded.*`,
 140                 },
 141                 {
 142                         path:  writePath,
 143                         cmd:   "move \"test &#!%20 file\" testfile\n",
 144                         match: `(?ms).*Moving .* succeeded.*`,
 145                 },
 146                 {
 147                         path:  writePath,
 148                         cmd:   "mkcol newdir0/\n",
 149                         match: `(?ms).*Creating .* succeeded.*`,
 150                 },
 151                 {
 152                         path:  writePath,
 153                         cmd:   "move testfile newdir0/\n",
 154                         match: `(?ms).*Moving .* succeeded.*`,
 155                 },
 156                 {
 157                         path:  writePath,
 158                         cmd:   "move testfile newdir0/\n",
 159                         match: `(?ms).*Moving .* failed.*`,
 160                 },
 161                 {
 162                         path:  writePath,
 163                         cmd:   "lock newdir0/testfile\n",
 164                         match: `(?ms).*Locking .* succeeded.*`,
 165                 },
 166                 {
 167                         path:  writePath,
 168                         cmd:   "unlock newdir0/testfile\nasdf\n",
 169                         match: `(?ms).*Unlocking .* succeeded.*`,
 170                 },
 171                 {
 172                         path:  writePath,
 173                         cmd:   "ls\n",
 174                         match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
 175                 },
 176                 {
 177                         path:  writePath,
 178                         cmd:   "move newdir0/testfile emptyfile/bogus/\n",
 179                         match: `(?ms).*Moving .* failed.*`,
 180                 },
 181                 {
 182                         path:  writePath,
 183                         cmd:   "mkcol newdir1\n",
 184                         match: `(?ms).*Creating .* succeeded.*`,
 185                 },
 186                 {
 187                         path:  writePath,
 188                         cmd:   "move newdir1/ newdir1x/\n",
 189                         match: `(?ms).*Moving .* succeeded.*`,
 190                 },
 191                 {
 192                         // This forces a refresh via needSync --
 193                         // otherwise, in the next "move" trial,
 194                         // cadaver will probe "PROPFIND newdir1",
 195                         // conclude newdir1 is an existing directory,
 196                         // and change our "move newdir1x newdir1"
 197                         // command to "move newdir1x
 198                         // newdir1/newdir1x".
 199                         path:  writePath,
 200                         cmd:   "get newdir1x/nofile " + checkfile.Name() + "\n",
 201                         match: `(?ms).*Downloading .* failed.*`,
 202                 },
 203                 {
 204                         path:  writePath,
 205                         cmd:   "move newdir1x newdir1\n",
 206                         match: `(?ms).*Moving .* succeeded.*`,
 207                 },
 208                 {
 209                         path:  writePath,
 210                         cmd:   "move newdir0/testfile newdir1/testfile\n",
 211                         match: `(?ms).*Moving .* succeeded.*`,
 212                 },
 213                 {
 214                         path:  writePath,
 215                         cmd:   "move newdir1 newdir1/newdir1\n",
 216                         match: `(?ms).*Moving .* failed.*`,
 217                 },
 218                 {
 219                         path:  writePath,
 220                         cmd:   "get newdir1/testfile '" + checkfile.Name() + "'\n",
 221                         match: `(?ms).*succeeded.*`,
 222                         data:  testdata,
 223                 },
 224                 {
 225                         path:  writePath,
 226                         cmd:   "put '" + localfile.Name() + "' newdir1/testfile1\n",
 227                         match: `(?ms).*Uploading .* succeeded.*`,
 228                 },
 229                 {
 230                         path:  writePath,
 231                         cmd:   "mkcol newdir2\n",
 232                         match: `(?ms).*Creating .* succeeded.*`,
 233                 },
 234                 {
 235                         path:  writePath,
 236                         cmd:   "put '" + localfile.Name() + "' newdir2/testfile2\n",
 237                         match: `(?ms).*Uploading .* succeeded.*`,
 238                 },
 239                 {
 240                         path:  writePath,
 241                         cmd:   "copy newdir2/testfile2 testfile3\n",
 242                         match: `(?ms).*succeeded.*`,
 243                 },
 244                 {
 245                         path:  writePath,
 246                         cmd:   "get testfile3 '" + checkfile.Name() + "'\n",
 247                         match: `(?ms).*succeeded.*`,
 248                         data:  testdata,
 249                 },
 250                 {
 251                         path:  writePath,
 252                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 253                         match: `(?ms).*succeeded.*`,
 254                         data:  testdata,
 255                 },
 256                 {
 257                         path:  writePath,
 258                         cmd:   "rmcol newdir2\n",
 259                         match: `(?ms).*Deleting collection .* succeeded.*`,
 260                 },
 261                 {
 262                         path:  writePath,
 263                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 264                         match: `(?ms).*Downloading .* failed.*`,
 265                 },
 266                 {
 267                         path:  "/c=" + arvadostest.UserAgreementCollection + "/t=" + arv.AuthToken + "/",
 268                         cmd:   "put '" + localfile.Name() + "' foo\n",
 269                         match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
 270                 },
 271                 {
 272                         path:  pdhPath,
 273                         cmd:   "put '" + localfile.Name() + "' foo\n",
 274                         match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
 275                 },
 276                 {
 277                         path:  pdhPath,
 278                         cmd:   "move foo bar\n",
 279                         match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
 280                 },
 281                 {
 282                         path:  pdhPath,
 283                         cmd:   "copy foo bar\n",
 284                         match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
 285                 },
 286                 {
 287                         path:  pdhPath,
 288                         cmd:   "delete foo\n",
 289                         match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
 290                 },
 291                 {
 292                         path:  pdhPath,
 293                         cmd:   "lock foo\n",
 294                         match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
 295                 },
 296         } {
 297                 c.Logf("=== %s trial %+v", s.testServer.URL, trial)
 298                 if skip != nil && skip(trial.path) {
 299                         c.Log("(skip)")
 300                         continue
 301                 }
 302
 303                 os.Remove(checkfile.Name())
 304
 305                 stdout := s.runCadaver(c, password, trial.path, trial.cmd)
 306                 c.Check(stdout, check.Matches, trial.match)
 307
 308                 if trial.data == "" && !trial.checkemptydata {
 309                         continue
 310                 }
 311                 checkfile, err = os.Open(checkfile.Name())
 312                 c.Assert(err, check.IsNil)
 313                 checkfile.Seek(0, os.SEEK_SET)
 314                 got, err := ioutil.ReadAll(checkfile)
 315                 c.Check(string(got), check.Equals, trial.data)
 316                 c.Check(err, check.IsNil)
 317         }
 318 }
 319
 320 func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
 321         for _, path := range []string{"/by_id", "/by_id/"} {
 322                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 323                 c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
 324         }
 325         for _, path := range []string{
 326                 "/by_id/" + arvadostest.FooCollectionPDH,
 327                 "/by_id/" + arvadostest.FooCollectionPDH + "/",
 328                 "/by_id/" + arvadostest.FooCollection,
 329                 "/by_id/" + arvadostest.FooCollection + "/",
 330         } {
 331                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 332                 c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
 333         }
 334 }
 335
 336 func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
 337         for _, path := range []string{"/"} {
 338                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 339                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
 340                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
 341         }
 342         for _, path := range []string{"/users", "/users/"} {
 343                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 344                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
 345         }
 346         for _, path := range []string{"/users/active", "/users/active/"} {
 347                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 348                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
 349                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
 350         }
 351         for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
 352                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 353                 c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
 354         }
 355 }
 356
 357 func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
 358         tempdir, err := ioutil.TempDir("", "keep-web-test-")
 359         c.Assert(err, check.IsNil)
 360         defer os.RemoveAll(tempdir)
 361
 362         cmd := exec.Command("cadaver", s.testServer.URL+path)
 363         if password != "" {
 364                 // cadaver won't try username/password authentication
 365                 // unless the server responds 401 to an
 366                 // unauthenticated request, which it only does in
 367                 // AttachmentOnlyHost, TrustAllContent, and
 368                 // per-collection vhost cases.
 369                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.URL[7:]
 370
 371                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 372                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 373                 c.Assert(err, check.IsNil)
 374                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 375                 c.Assert(err, check.IsNil)
 376                 c.Assert(f.Close(), check.IsNil)
 377         }
 378         cmd.Stdin = bytes.NewBufferString(stdin)
 379         stdout, err := cmd.StdoutPipe()
 380         c.Assert(err, check.Equals, nil)
 381         cmd.Stderr = cmd.Stdout
 382         go cmd.Start()
 383
 384         var buf bytes.Buffer
 385         _, err = io.Copy(&buf, stdout)
 386         c.Check(err, check.Equals, nil)
 387         err = cmd.Wait()
 388         c.Check(err, check.Equals, nil)
 389         return buf.String()
 390 }