services/keep-web/cadaver_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "strings"
  16         "time"
  17
  18         "git.arvados.org/arvados.git/sdk/go/arvados"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         check "gopkg.in/check.v1"
  21 )
  22
  23 func (s *IntegrationSuite) TestCadaverHTTPAuth(c *check.C) {
  24         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  25                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/"
  26                 w := "/c=" + newCollection.UUID + "/"
  27                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  28                 return r, w, pdh
  29         }, nil)
  30 }
  31
  32 func (s *IntegrationSuite) TestCadaverPathAuth(c *check.C) {
  33         s.testCadaver(c, "", func(newCollection arvados.Collection) (string, string, string) {
  34                 r := "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/"
  35                 w := "/c=" + newCollection.UUID + "/t=" + arvadostest.ActiveToken + "/"
  36                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/t=" + arvadostest.ActiveToken + "/"
  37                 return r, w, pdh
  38         }, nil)
  39 }
  40
  41 func (s *IntegrationSuite) TestCadaverUserProject(c *check.C) {
  42         rpath := "/users/active/foo_file_in_dir/"
  43         s.testCadaver(c, arvadostest.ActiveToken, func(newCollection arvados.Collection) (string, string, string) {
  44                 wpath := "/users/active/" + newCollection.Name
  45                 pdh := "/c=" + strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + "/"
  46                 return rpath, wpath, pdh
  47         }, func(path string) bool {
  48                 // Skip tests that rely on writes, because /users/
  49                 // tree is read-only.
  50                 return !strings.HasPrefix(path, rpath) || strings.HasPrefix(path, rpath+"_/")
  51         })
  52 }
  53
  54 func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
  55         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  56
  57         testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
  58
  59         tempdir, err := ioutil.TempDir("", "keep-web-test-")
  60         c.Assert(err, check.IsNil)
  61         defer os.RemoveAll(tempdir)
  62
  63         localfile, err := ioutil.TempFile(tempdir, "localfile")
  64         c.Assert(err, check.IsNil)
  65         localfile.Write(testdata)
  66
  67         emptyfile, err := ioutil.TempFile(tempdir, "emptyfile")
  68         c.Assert(err, check.IsNil)
  69
  70         checkfile, err := ioutil.TempFile(tempdir, "checkfile")
  71         c.Assert(err, check.IsNil)
  72
  73         var newCollection arvados.Collection
  74         arv := arvados.NewClientFromEnv()
  75         arv.AuthToken = arvadostest.ActiveToken
  76         err = arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{}})
  77         c.Assert(err, check.IsNil)
  78
  79         readPath, writePath, pdhPath := pathFunc(newCollection)
  80
  81         matchToday := time.Now().Format("Jan +2")
  82
  83         type testcase struct {
  84                 path  string
  85                 cmd   string
  86                 match string
  87                 data  []byte
  88         }
  89         for _, trial := range []testcase{
  90                 {
  91                         path:  readPath,
  92                         cmd:   "ls\n",
  93                         match: `(?ms).*dir1 *0 .*`,
  94                 },
  95                 {
  96                         path:  readPath,
  97                         cmd:   "ls dir1\n",
  98                         match: `(?ms).*bar *3.*foo *3 .*`,
  99                 },
 100                 {
 101                         path:  readPath + "_/dir1",
 102                         cmd:   "ls\n",
 103                         match: `(?ms).*bar *3.*foo *3 .*`,
 104                 },
 105                 {
 106                         path:  readPath + "dir1/",
 107                         cmd:   "ls\n",
 108                         match: `(?ms).*bar *3.*foo +3 +Feb +\d+ +2014.*`,
 109                 },
 110                 {
 111                         path:  writePath,
 112                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 113                         match: `(?ms).*Not Found.*`,
 114                 },
 115                 {
 116                         path:  writePath,
 117                         cmd:   "put '" + emptyfile.Name() + "' emptyfile\n",
 118                         match: `(?ms).*Uploading .* succeeded.*`,
 119                 },
 120                 {
 121                         path:  writePath,
 122                         cmd:   "get emptyfile '" + checkfile.Name() + "'\n",
 123                         match: `(?ms).*Downloading .* succeeded.*`,
 124                         data:  []byte{},
 125                 },
 126                 {
 127                         path:  writePath,
 128                         cmd:   "put '" + localfile.Name() + "' testfile\n",
 129                         match: `(?ms).*Uploading .* succeeded.*`,
 130                 },
 131                 {
 132                         path:  writePath,
 133                         cmd:   "get testfile '" + checkfile.Name() + "'\n",
 134                         match: `(?ms).*succeeded.*`,
 135                         data:  testdata,
 136                 },
 137                 {
 138                         path:  writePath,
 139                         cmd:   "move testfile \"test &#!%20 file\"\n",
 140                         match: `(?ms).*Moving .* succeeded.*`,
 141                 },
 142                 {
 143                         path:  writePath,
 144                         cmd:   "move \"test &#!%20 file\" testfile\n",
 145                         match: `(?ms).*Moving .* succeeded.*`,
 146                 },
 147                 {
 148                         path:  writePath,
 149                         cmd:   "move testfile newdir0/\n",
 150                         match: `(?ms).*Moving .* succeeded.*`,
 151                 },
 152                 {
 153                         path:  writePath,
 154                         cmd:   "move testfile newdir0/\n",
 155                         match: `(?ms).*Moving .* failed.*`,
 156                 },
 157                 {
 158                         path:  writePath,
 159                         cmd:   "lock newdir0/testfile\n",
 160                         match: `(?ms).*Locking .* succeeded.*`,
 161                 },
 162                 {
 163                         path:  writePath,
 164                         cmd:   "unlock newdir0/testfile\nasdf\n",
 165                         match: `(?ms).*Unlocking .* succeeded.*`,
 166                 },
 167                 {
 168                         path:  writePath,
 169                         cmd:   "ls\n",
 170                         match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
 171                 },
 172                 {
 173                         path:  writePath,
 174                         cmd:   "move newdir0/testfile emptyfile/bogus/\n",
 175                         match: `(?ms).*Moving .* failed.*`,
 176                 },
 177                 {
 178                         path:  writePath,
 179                         cmd:   "mkcol newdir1\n",
 180                         match: `(?ms).*Creating .* succeeded.*`,
 181                 },
 182                 {
 183                         path:  writePath,
 184                         cmd:   "move newdir1/ newdir1x/\n",
 185                         match: `(?ms).*Moving .* succeeded.*`,
 186                 },
 187                 {
 188                         path:  writePath,
 189                         cmd:   "move newdir1x newdir1\n",
 190                         match: `(?ms).*Moving .* succeeded.*`,
 191                 },
 192                 {
 193                         path:  writePath,
 194                         cmd:   "move newdir0/testfile newdir1/\n",
 195                         match: `(?ms).*Moving .* succeeded.*`,
 196                 },
 197                 {
 198                         path:  writePath,
 199                         cmd:   "move newdir1 newdir1/\n",
 200                         match: `(?ms).*Moving .* failed.*`,
 201                 },
 202                 {
 203                         path:  writePath,
 204                         cmd:   "get newdir1/testfile '" + checkfile.Name() + "'\n",
 205                         match: `(?ms).*succeeded.*`,
 206                         data:  testdata,
 207                 },
 208                 {
 209                         path:  writePath,
 210                         cmd:   "put '" + localfile.Name() + "' newdir1/testfile1\n",
 211                         match: `(?ms).*Uploading .* succeeded.*`,
 212                 },
 213                 {
 214                         path:  writePath,
 215                         cmd:   "mkcol newdir2\n",
 216                         match: `(?ms).*Creating .* succeeded.*`,
 217                 },
 218                 {
 219                         path:  writePath,
 220                         cmd:   "put '" + localfile.Name() + "' newdir2/testfile2\n",
 221                         match: `(?ms).*Uploading .* succeeded.*`,
 222                 },
 223                 {
 224                         path:  writePath,
 225                         cmd:   "copy newdir2/testfile2 testfile3\n",
 226                         match: `(?ms).*succeeded.*`,
 227                 },
 228                 {
 229                         path:  writePath,
 230                         cmd:   "get testfile3 '" + checkfile.Name() + "'\n",
 231                         match: `(?ms).*succeeded.*`,
 232                         data:  testdata,
 233                 },
 234                 {
 235                         path:  writePath,
 236                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 237                         match: `(?ms).*succeeded.*`,
 238                         data:  testdata,
 239                 },
 240                 {
 241                         path:  writePath,
 242                         cmd:   "rmcol newdir2\n",
 243                         match: `(?ms).*Deleting collection .* succeeded.*`,
 244                 },
 245                 {
 246                         path:  writePath,
 247                         cmd:   "get newdir2/testfile2 '" + checkfile.Name() + "'\n",
 248                         match: `(?ms).*Downloading .* failed.*`,
 249                 },
 250                 {
 251                         path:  "/c=" + arvadostest.UserAgreementCollection + "/t=" + arv.AuthToken + "/",
 252                         cmd:   "put '" + localfile.Name() + "' foo\n",
 253                         match: `(?ms).*Uploading .* failed:.*403 Forbidden.*`,
 254                 },
 255                 {
 256                         path:  pdhPath,
 257                         cmd:   "put '" + localfile.Name() + "' foo\n",
 258                         match: `(?ms).*Uploading .* failed:.*405 Method Not Allowed.*`,
 259                 },
 260                 {
 261                         path:  pdhPath,
 262                         cmd:   "move foo bar\n",
 263                         match: `(?ms).*Moving .* failed:.*405 Method Not Allowed.*`,
 264                 },
 265                 {
 266                         path:  pdhPath,
 267                         cmd:   "copy foo bar\n",
 268                         match: `(?ms).*Copying .* failed:.*405 Method Not Allowed.*`,
 269                 },
 270                 {
 271                         path:  pdhPath,
 272                         cmd:   "delete foo\n",
 273                         match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
 274                 },
 275                 {
 276                         path:  pdhPath,
 277                         cmd:   "lock foo\n",
 278                         match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
 279                 },
 280         } {
 281                 c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
 282                 if skip != nil && skip(trial.path) {
 283                         c.Log("(skip)")
 284                         continue
 285                 }
 286
 287                 os.Remove(checkfile.Name())
 288
 289                 stdout := s.runCadaver(c, password, trial.path, trial.cmd)
 290                 c.Check(stdout, check.Matches, trial.match)
 291
 292                 if trial.data == nil {
 293                         continue
 294                 }
 295                 checkfile, err = os.Open(checkfile.Name())
 296                 c.Assert(err, check.IsNil)
 297                 checkfile.Seek(0, os.SEEK_SET)
 298                 got, err := ioutil.ReadAll(checkfile)
 299                 c.Check(got, check.DeepEquals, trial.data)
 300                 c.Check(err, check.IsNil)
 301         }
 302 }
 303
 304 func (s *IntegrationSuite) TestCadaverByID(c *check.C) {
 305         for _, path := range []string{"/by_id", "/by_id/"} {
 306                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 307                 c.Check(stdout, check.Matches, `(?ms).*collection is empty.*`)
 308         }
 309         for _, path := range []string{
 310                 "/by_id/" + arvadostest.FooCollectionPDH,
 311                 "/by_id/" + arvadostest.FooCollectionPDH + "/",
 312                 "/by_id/" + arvadostest.FooCollection,
 313                 "/by_id/" + arvadostest.FooCollection + "/",
 314         } {
 315                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 316                 c.Check(stdout, check.Matches, `(?ms).*\s+foo\s+3 .*`)
 317         }
 318 }
 319
 320 func (s *IntegrationSuite) TestCadaverUsersDir(c *check.C) {
 321         for _, path := range []string{"/"} {
 322                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 323                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+by_id\s+0 .*`)
 324                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+users\s+0 .*`)
 325         }
 326         for _, path := range []string{"/users", "/users/"} {
 327                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 328                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+active.*`)
 329         }
 330         for _, path := range []string{"/users/active", "/users/active/"} {
 331                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 332                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+A Project\s+0 .*`)
 333                 c.Check(stdout, check.Matches, `(?ms).*Coll:\s+bar_file\s+0 .*`)
 334         }
 335         for _, path := range []string{"/users/admin", "/users/doesnotexist", "/users/doesnotexist/"} {
 336                 stdout := s.runCadaver(c, arvadostest.ActiveToken, path, "ls")
 337                 c.Check(stdout, check.Matches, `(?ms).*404 Not Found.*`)
 338         }
 339 }
 340
 341 func (s *IntegrationSuite) runCadaver(c *check.C, password, path, stdin string) string {
 342         tempdir, err := ioutil.TempDir("", "keep-web-test-")
 343         c.Assert(err, check.IsNil)
 344         defer os.RemoveAll(tempdir)
 345
 346         cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
 347         if password != "" {
 348                 // cadaver won't try username/password authentication
 349                 // unless the server responds 401 to an
 350                 // unauthenticated request, which it only does in
 351                 // AttachmentOnlyHost, TrustAllContent, and
 352                 // per-collection vhost cases.
 353                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
 354
 355                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 356                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 357                 c.Assert(err, check.IsNil)
 358                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 359                 c.Assert(err, check.IsNil)
 360                 c.Assert(f.Close(), check.IsNil)
 361         }
 362         cmd.Stdin = bytes.NewBufferString(stdin)
 363         stdout, err := cmd.StdoutPipe()
 364         c.Assert(err, check.Equals, nil)
 365         cmd.Stderr = cmd.Stdout
 366         go cmd.Start()
 367
 368         var buf bytes.Buffer
 369         _, err = io.Copy(&buf, stdout)
 370         c.Check(err, check.Equals, nil)
 371         err = cmd.Wait()
 372         c.Check(err, check.Equals, nil)
 373         return buf.String()
 374 }