services/keep-web/cadaver_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "strings"
  16         "time"
  17
  18         "git.arvados.org/arvados.git/sdk/go/arvados"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
  24         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  25                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
  26                 w := "/c=" + newCollection.UUID + "/"
  27                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  28                 return r, w, pdh
  29         }, nil)
  30 }
  31
  32 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
  33         s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
  34                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
  35                 w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
  36                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
  37                 return r, w, pdh
  38         }, nil)
  39 }
  40
  41 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
  42         rpath := "/users/active/foo_file_in_dir/"
  43         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  44                 wpath := "/users/active/" + newCollection.Name
  45                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  46                 return rpath, wpath, pdh
  47         }, func(path string) bool {
  48                 // Skip tests that rely on writes, because /users/
  49                 // tree is read-only.
  50                 return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
  51         })
  52 }
  53
  54 func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
  55         testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
  56
  57         tempdir, err := ioutil.TempDir("", "keep-web-test-")
  58         c.Assert(err, check.IsNil)
  59         defer os.RemoveAll(tempdir)
  60
  61         localfile, err := ioutil.TempFile(tempdir, "localfile")
  62         c.Assert(err, check.IsNil)
  63         localfile.Write(testdata)
  64
  65         emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
  66         c.Assert(err, check.IsNil)
  67
  68         checkfile, err := ioutil.TempFile(tempdir, "checkfile")
  69         c.Assert(err, check.IsNil)
  70
  71         var newCollection arvados.Collection
  72         arv := arvados.NewClientFromEnv()
  73         arv.AuthToken = arvadostest.ActiveToken
  74         err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{}})
  75         c.Assert(err, check.IsNil)
  76
  77         readPath, writePath, pdhPath := pathFunc(newCollection)
  78
  79         matchToday := time.Now().Format("Jan +2")
  80
  81         type testcase struct {
  82                 path  string
  83                 cmd   string
  84                 match string
  85                 data  []byte
  86         }
  87         for _, trial := range []testcase{
  88                 {
  89                         path:  readPath,
  90                         cmd:   "ls\n",
  91                         match: `(?ms).*dir1 *0 .*`,
  92                 },
  93                 {
  94                         path:  readPath,
  95                         cmd:   "ls dir1\n",
  96                         match: `(?ms).*bar *3.*foo *3 .*`,
  97                 },
  98                 {
  99                         path:  readPath + "_/dir1",
 100                         cmd:   "ls\n",
 101                         match: `(?ms).*bar *3.*foo *3 .*`,
 102                 },
 103                 {
 104                         path:  readPath + "dir1/",
 105                         cmd:   "ls\n",
 106                         match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
 107                 },
 108                 {
 109                         path:  writePath,
 110                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 111                         match: `(?ms).*Not Found.*`,
 112                 },
 113                 {
 114                         path:  writePath,
 115                         cmd:   "put '" + emptyfile.Name() + "' emptyfile\n",
 116                         match: `(?ms).*Uploading .* succeeded.*`,
 117                 },
 118                 {
 119                         path:  writePath,
 120                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 121                         match: `(?ms).*Downloading .* succeeded.*`,
 122                         data:  []byte{},
 123                 },
 124                 {
 125                         path:  writePath,
 126                         cmd:   "put '" + localfile.Name() + "' testfile\n",
 127                         match: `(?ms).*Uploading .* succeeded.*`,
 128                 },
 129                 {
 130                         path:  writePath,
 131                         cmd:   "get testfile '" + checkfile.Name() + "'\n",
 132                         match: `(?ms).*succeeded.*`,
 133                         data:  testdata,
 134                 },
 135                 {
 136                         path:  writePath,
 137                         cmd:   "move testfile \"test &#!%20 file\"\n",
 138                         match: `(?ms).*Moving .* succeeded.*`,
 139                 },
 140                 {
 141                         path:  writePath,
 142                         cmd:   "move \"test &#!%20 file\" testfile\n",
 143                         match: `(?ms).*Moving .* succeeded.*`,
 144                 },
 145                 {
 146                         path:  writePath,
 147                         cmd:   "mkcol newdir0/\n",
 148                         match: `(?ms).*Creating .* succeeded.*`,
 149                 },
 150                 {
 151                         path:  writePath,
 152                         cmd:   "move testfile newdir0/\n",
 153                         match: `(?ms).*Moving .* succeeded.*`,
 154                 },
 155                 {
 156                         path:  writePath,
 157                         cmd:   "move testfile newdir0/\n",
 158                         match: `(?ms).*Moving .* failed.*`,
 159                 },
 160                 {
 161                         path:  writePath,
 162                         cmd:   "lock newdir0/testfile\n",
 163                         match: `(?ms).*Locking .* succeeded.*`,
 164                 },
 165                 {
 166                         path:  writePath,
 167                         cmd:   "unlock newdir0/testfile\nasdf\n",
 168                         match: `(?ms).*Unlocking .* succeeded.*`,
 169                 },
 170                 {
 171                         path:  writePath,
 172                         cmd:   "ls\n",
 173                         match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
 174                 },
 175                 {
 176                         path:  writePath,
 177                         cmd:   "move newdir0/testfile emptyfile/bogus/\n",
 178                         match: `(?ms).*Moving .* failed.*`,
 179                 },
 180                 {
 181                         path:  writePath,
 182                         cmd:   "mkcol newdir1\n",
 183                         match: `(?ms).*Creating .* succeeded.*`,
 184                 },
 185                 {
 186                         path:  writePath,
 187                         cmd:   "move newdir1/ newdir1x/\n",
 188                         match: `(?ms).*Moving .* succeeded.*`,
 189                 },
 190                 {
 191                         path:  writePath,
 192                         cmd:   "move newdir1x newdir1\n",
 193                         match: `(?ms).*Moving .* succeeded.*`,
 194                 },
 195                 {
 196                         path:  writePath,
 197                         cmd:   "move newdir0/testfile newdir1/\n",
 198                         match: `(?ms).*Moving .* succeeded.*`,
 199                 },
 200                 {
 201                         path:  writePath,
 202                         cmd:   "move newdir1 newdir1/\n",
 203                         match: `(?ms).*Moving .* failed.*`,
 204                 },
 205                 {
 206                         path:  writePath,
 207                         cmd:   "get newdir1/testfile '" + checkfile.Name() + "'\n",
 208                         match: `(?ms).*succeeded.*`,
 209                         data:  testdata,
 210                 },
 211                 {
 212                         path:  writePath,
 213                         cmd:   "put '" + localfile.Name() + "' newdir1/testfile1\n",
 214                         match: `(?ms).*Uploading .* succeeded.*`,
 215                 },
 216                 {
 217                         path:  writePath,
 218                         cmd:   "mkcol newdir2\n",
 219                         match: `(?ms).*Creating .* succeeded.*`,
 220                 },
 221                 {
 222                         path:  writePath,
 223                         cmd:   "put '" + localfile.Name() + "' newdir2/testfile2\n",
 224                         match: `(?ms).*Uploading .* succeeded.*`,
 225                 },
 226                 {
 227                         path:  writePath,
 228                         cmd:   "copy newdir2/testfile2 testfile3\n",
 229                         match: `(?ms).*succeeded.*`,
 230                 },
 231                 {
 232                         path:  writePath,
 233                         cmd:   "get testfile3 '" + checkfile.Name() + "'\n",
 234                         match: `(?ms).*succeeded.*`,
 235                         data:  testdata,
 236                 },
 237                 {
 238                         path:  writePath,
 239                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 240                         match: `(?ms).*succeeded.*`,
 241                         data:  testdata,
 242                 },
 243                 {
 244                         path:  writePath,
 245                         cmd:   "rmcol newdir2\n",
 246                         match: `(?ms).*Deleting collection .* succeeded.*`,
 247                 },
 248                 {
 249                         path:  writePath,
 250                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 251                         match: `(?ms).*Downloading .* failed.*`,
 252                 },
 253                 {
 254                         path:  "/c=" + arvadostest.UserAgreementCollection + "/t=" + arv.AuthToken + "/",
 255                         cmd:   "put '" + localfile.Name() + "' foo\n",
 256                         match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
 257                 },
 258                 {
 259                         path:  pdhPath,
 260                         cmd:   "put '" + localfile.Name() + "' foo\n",
 261                         match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
 262                 },
 263                 {
 264                         path:  pdhPath,
 265                         cmd:   "move foo bar\n",
 266                         match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
 267                 },
 268                 {
 269                         path:  pdhPath,
 270                         cmd:   "copy foo bar\n",
 271                         match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
 272                 },
 273                 {
 274                         path:  pdhPath,
 275                         cmd:   "delete foo\n",
 276                         match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
 277                 },
 278                 {
 279                         path:  pdhPath,
 280                         cmd:   "lock foo\n",
 281                         match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
 282                 },
 283         } {
 284                 c.Logf("%s %+v", s.testServer.URL, trial)
 285                 if skip != nil && skip(trial.path) {
 286                         c.Log("(skip)")
 287                         continue
 288                 }
 289
 290                 os.Remove(checkfile.Name())
 291
 292                 stdout := s.runCadaver(c, password, trial.path, trial.cmd)
 293                 c.Check(stdout, check.Matches, trial.match)
 294
 295                 if trial.data == nil {
 296                         continue
 297                 }
 298                 checkfile, err = os.Open(checkfile.Name())
 299                 c.Assert(err, check.IsNil)
 300                 checkfile.Seek(0, os.SEEK_SET)
 301                 got, err := ioutil.ReadAll(checkfile)
 302                 c.Check(got, check.DeepEquals, trial.data)
 303                 c.Check(err, check.IsNil)
 304         }
 305 }
 306
 307 func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
 308         for _, path := range []string{"/by_id", "/by_id/"} {
 309                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 310                 c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
 311         }
 312         for _, path := range []string{
 313                 "/by_id/" + arvadostest.FooCollectionPDH,
 314                 "/by_id/" + arvadostest.FooCollectionPDH + "/",
 315                 "/by_id/" + arvadostest.FooCollection,
 316                 "/by_id/" + arvadostest.FooCollection + "/",
 317         } {
 318                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 319                 c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
 320         }
 321 }
 322
 323 func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
 324         for _, path := range []string{"/"} {
 325                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 326                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
 327                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
 328         }
 329         for _, path := range []string{"/users", "/users/"} {
 330                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 331                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
 332         }
 333         for _, path := range []string{"/users/active", "/users/active/"} {
 334                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 335                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
 336                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
 337         }
 338         for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
 339                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 340                 c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
 341         }
 342 }
 343
 344 func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
 345         tempdir, err := ioutil.TempDir("", "keep-web-test-")
 346         c.Assert(err, check.IsNil)
 347         defer os.RemoveAll(tempdir)
 348
 349         cmd := exec.Command("cadaver", s.testServer.URL+path)
 350         if password != "" {
 351                 // cadaver won't try username/password authentication
 352                 // unless the server responds 401 to an
 353                 // unauthenticated request, which it only does in
 354                 // AttachmentOnlyHost, TrustAllContent, and
 355                 // per-collection vhost cases.
 356                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.URL[7:]
 357
 358                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 359                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 360                 c.Assert(err, check.IsNil)
 361                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 362                 c.Assert(err, check.IsNil)
 363                 c.Assert(f.Close(), check.IsNil)
 364         }
 365         cmd.Stdin = bytes.NewBufferString(stdin)
 366         stdout, err := cmd.StdoutPipe()
 367         c.Assert(err, check.Equals, nil)
 368         cmd.Stderr = cmd.Stdout
 369         go cmd.Start()
 370
 371         var buf bytes.Buffer
 372         _, err = io.Copy(&buf, stdout)
 373         c.Check(err, check.Equals, nil)
 374         err = cmd.Wait()
 375         c.Check(err, check.Equals, nil)
 376         return buf.String()
 377 }