lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11         "net/url"
  12         "os"
  13         "testing"
  14
  15         "git.curoverse.com/arvados.git/lib/controller/router"
  16         "git.curoverse.com/arvados.git/lib/controller/rpc"
  17         "git.curoverse.com/arvados.git/sdk/go/arvados"
  18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  19         "git.curoverse.com/arvados.git/sdk/go/auth"
  20         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  21         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 // Gocheck boilerplate
  26 func Test(t *testing.T) {
  27         check.TestingT(t)
  28 }
  29
  30 var (
  31         _ = check.Suite(&FederationSuite{})
  32         _ = check.Suite(&CollectionListSuite{})
  33 )
  34
  35 type FederationSuite struct {
  36         cluster *arvados.Cluster
  37         ctx     context.Context
  38         fed     *Conn
  39 }
  40
  41 func (s *FederationSuite) SetUpTest(c *check.C) {
  42         s.cluster = &arvados.Cluster{
  43                 ClusterID: "aaaaa",
  44                 RemoteClusters: map[string]arvados.RemoteCluster{
  45                         "aaaaa": arvados.RemoteCluster{
  46                                 Host: os.Getenv("ARVADOS_API_HOST"),
  47                         },
  48                 },
  49         }
  50         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  51         s.cluster.TLS.Insecure = true
  52         s.cluster.API.MaxItemsPerResponse = 3
  53
  54         ctx := context.Background()
  55         ctx = ctxlog.Context(ctx, ctxlog.TestLogger(c))
  56         ctx = auth.NewContext(ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  57         s.ctx = ctx
  58
  59         s.fed = New(s.cluster)
  60 }
  61
  62 func (s *FederationSuite) addDirectRemote(c *check.C, id string, backend backend) {
  63         s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
  64                 Host: "in-process.local",
  65         }
  66         s.fed.remotes[id] = backend
  67 }
  68
  69 func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend backend) {
  70         srv := httpserver.Server{Addr: ":"}
  71         srv.Handler = router.New(backend)
  72         c.Check(srv.Start(), check.IsNil)
  73         s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
  74                 Host:  srv.Addr,
  75                 Proxy: true,
  76         }
  77         s.fed.remotes[id] = rpc.NewConn(id, &url.URL{Scheme: "http", Host: srv.Addr}, true, saltedTokenProvider(s.fed.local, id))
  78 }
  79
  80 type collectionLister struct {
  81         arvadostest.APIStub
  82         ItemsToReturn []arvados.Collection
  83         MaxPageSize   int
  84 }
  85
  86 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  87 nextfilter:
  88         for _, f := range filters {
  89                 if f.Attr == "uuid" && f.Operator == "=" {
  90                         s, ok := f.Operand.(string)
  91                         if ok && s == c.UUID {
  92                                 continue nextfilter
  93                         }
  94                 } else if f.Attr == "uuid" && f.Operator == "in" {
  95                         if operand, ok := f.Operand.([]string); ok {
  96                                 for _, s := range operand {
  97                                         if s == c.UUID {
  98                                                 continue nextfilter
  99                                         }
 100                                 }
 101                         } else if operand, ok := f.Operand.([]interface{}); ok {
 102                                 for _, s := range operand {
 103                                         if s, ok := s.(string); ok && s == c.UUID {
 104                                                 continue nextfilter
 105                                         }
 106                                 }
 107                         }
 108                 }
 109                 return false
 110         }
 111         return true
 112 }
 113
 114 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
 115         cl.APIStub.CollectionList(ctx, options)
 116         for _, c := range cl.ItemsToReturn {
 117                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
 118                         break
 119                 }
 120                 if options.Limit >= 0 && len(resp.Items) >= options.Limit {
 121                         break
 122                 }
 123                 if cl.matchFilters(c, options.Filters) {
 124                         resp.Items = append(resp.Items, c)
 125                 }
 126         }
 127         return
 128 }
 129
 130 type CollectionListSuite struct {
 131         FederationSuite
 132         ids      []string   // aaaaa, bbbbb, ccccc
 133         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
 134         backends []*collectionLister
 135 }
 136
 137 func (s *CollectionListSuite) SetUpTest(c *check.C) {
 138         s.FederationSuite.SetUpTest(c)
 139
 140         s.ids = nil
 141         s.uuids = nil
 142         s.backends = nil
 143         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
 144                 cl := &collectionLister{}
 145                 s.ids = append(s.ids, id)
 146                 s.uuids = append(s.uuids, nil)
 147                 for j := 0; j < 5; j++ {
 148                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
 149                         s.uuids[i] = append(s.uuids[i], uuid)
 150                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
 151                                 UUID: uuid,
 152                         })
 153                 }
 154                 s.backends = append(s.backends, cl)
 155                 if i == 0 {
 156                         s.fed.local = cl
 157                 } else if i%1 == 0 {
 158                         // call some backends directly via API
 159                         s.addDirectRemote(c, id, cl)
 160                 } else {
 161                         // call some backends through rpc->router->API
 162                         // to ensure nothing is lost in translation
 163                         s.addHTTPRemote(c, id, cl)
 164                 }
 165         }
 166 }
 167
 168 type listTrial struct {
 169         count        string
 170         limit        int
 171         offset       int
 172         order        []string
 173         filters      []arvados.Filter
 174         expectUUIDs  []string
 175         expectCalls  []int // number of API calls to backends
 176         expectStatus int
 177 }
 178
 179 func (s *CollectionListSuite) TestCollectionListNoUUIDFilters(c *check.C) {
 180         s.test(c, listTrial{
 181                 count:       "none",
 182                 limit:       1,
 183                 expectUUIDs: []string{s.uuids[0][0]},
 184                 expectCalls: []int{1, 0, 0},
 185         })
 186 }
 187
 188 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 189         s.test(c, listTrial{
 190                 count:       "none",
 191                 limit:       -1,
 192                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 193                 expectUUIDs: []string{s.uuids[0][0]},
 194                 expectCalls: []int{1, 0, 0},
 195         })
 196 }
 197
 198 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 199         s.test(c, listTrial{
 200                 count:       "none",
 201                 limit:       -1,
 202                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 203                 expectUUIDs: []string{s.uuids[1][0]},
 204                 expectCalls: []int{0, 1, 0},
 205         })
 206 }
 207
 208 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 209         s.test(c, listTrial{
 210                 count:       "none",
 211                 limit:       -1,
 212                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 213                 expectUUIDs: []string{s.uuids[0][0]},
 214                 expectCalls: []int{1, 0, 0},
 215         })
 216 }
 217
 218 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 219         s.test(c, listTrial{
 220                 count:       "none",
 221                 limit:       -1,
 222                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 223                 expectUUIDs: []string{s.uuids[1][1]},
 224                 expectCalls: []int{0, 1, 0},
 225         })
 226 }
 227
 228 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 229         s.test(c, listTrial{
 230                 count:       "none",
 231                 limit:       -1,
 232                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 233                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 234                 expectCalls: []int{1, 1, 0},
 235         })
 236 }
 237
 238 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 239         s.test(c, listTrial{
 240                 count:       "none",
 241                 limit:       -1,
 242                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 243                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 244                 expectCalls: []int{0, 1, 1},
 245         })
 246 }
 247
 248 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 249         s.cluster.API.MaxItemsPerResponse = 2
 250         s.test(c, listTrial{
 251                 count: "none",
 252                 limit: -1,
 253                 filters: []arvados.Filter{
 254                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 255                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 256                 },
 257                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 258                 expectCalls: []int{1, 0, 1},
 259         })
 260 }
 261
 262 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 263         s.test(c, listTrial{
 264                 count:       "none",
 265                 limit:       -1,
 266                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 267                 expectUUIDs: []string{},
 268                 expectCalls: []int{0, 0, 0},
 269         })
 270 }
 271
 272 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 273         s.test(c, listTrial{
 274                 count: "none",
 275                 limit: -1,
 276                 filters: []arvados.Filter{
 277                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 278                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 279                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 280                 },
 281                 expectUUIDs: []string{s.uuids[0][0]},
 282                 expectCalls: []int{1, 0, 0},
 283         })
 284 }
 285
 286 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 287         for i := range s.backends {
 288                 s.uuids[i] = s.uuids[i][:3]
 289                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 290         }
 291         s.cluster.API.MaxItemsPerResponse = 9
 292         for _, stub := range s.backends {
 293                 stub.MaxPageSize = 2
 294         }
 295         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 296         s.test(c, listTrial{
 297                 count:       "none",
 298                 limit:       -1,
 299                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 300                 expectUUIDs: allUUIDs,
 301                 expectCalls: []int{2, 2, 2},
 302         })
 303 }
 304
 305 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 306         // not [yet] supported
 307         s.test(c, listTrial{
 308                 count: "none",
 309                 limit: -1,
 310                 filters: []arvados.Filter{
 311                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 312                         {"uuid", "is_a", "teapot"},
 313                 },
 314                 expectCalls:  []int{0, 0, 0},
 315                 expectStatus: http.StatusBadRequest,
 316         })
 317 }
 318
 319 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 320         for _, count := range []string{"", "exact"} {
 321                 s.test(c, listTrial{
 322                         count: count,
 323                         limit: -1,
 324                         filters: []arvados.Filter{
 325                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 326                                 {"uuid", "is_a", "teapot"},
 327                         },
 328                         expectCalls:  []int{0, 0, 0},
 329                         expectStatus: http.StatusBadRequest,
 330                 })
 331         }
 332 }
 333
 334 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 335         for _, limit := range []int{0, 1, 2} {
 336                 s.test(c, listTrial{
 337                         count: "none",
 338                         limit: limit,
 339                         filters: []arvados.Filter{
 340                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 341                                 {"uuid", "is_a", "teapot"},
 342                         },
 343                         expectCalls:  []int{0, 0, 0},
 344                         expectStatus: http.StatusBadRequest,
 345                 })
 346         }
 347 }
 348
 349 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 350         s.test(c, listTrial{
 351                 count:  "none",
 352                 limit:  -1,
 353                 offset: 1,
 354                 filters: []arvados.Filter{
 355                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 356                         {"uuid", "is_a", "teapot"},
 357                 },
 358                 expectCalls:  []int{0, 0, 0},
 359                 expectStatus: http.StatusBadRequest,
 360         })
 361 }
 362
 363 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 364         s.test(c, listTrial{
 365                 count: "none",
 366                 limit: -1,
 367                 order: []string{"uuid desc"},
 368                 filters: []arvados.Filter{
 369                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 370                         {"uuid", "is_a", "teapot"},
 371                 },
 372                 expectCalls:  []int{0, 0, 0},
 373                 expectStatus: http.StatusBadRequest,
 374         })
 375 }
 376
 377 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 378         s.test(c, listTrial{
 379                 count: "none",
 380                 limit: -1,
 381                 filters: []arvados.Filter{
 382                         {"uuid", "in", "teapot"},
 383                 },
 384                 expectCalls:  []int{0, 0, 0},
 385                 expectStatus: http.StatusBadRequest,
 386         })
 387 }
 388
 389 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 390         s.test(c, listTrial{
 391                 count: "none",
 392                 limit: -1,
 393                 filters: []arvados.Filter{
 394                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 395                 },
 396                 expectStatus: http.StatusNotFound,
 397         })
 398 }
 399
 400 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 401         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{})
 402         s.test(c, listTrial{
 403                 count: "none",
 404                 limit: -1,
 405                 filters: []arvados.Filter{
 406                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 407                 },
 408                 expectStatus: http.StatusBadGateway,
 409         })
 410 }
 411
 412 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 413         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 414                 Count:   trial.count,
 415                 Limit:   trial.limit,
 416                 Offset:  trial.offset,
 417                 Order:   trial.order,
 418                 Filters: trial.filters,
 419         })
 420         if trial.expectStatus != 0 {
 421                 c.Assert(err, check.NotNil)
 422                 err, _ := err.(interface{ HTTPStatus() int })
 423                 c.Assert(err, check.NotNil) // err must implement HTTPStatus()
 424                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 425                 c.Logf("returned error is %#v", err)
 426                 c.Logf("returned error string is %q", err)
 427         } else {
 428                 c.Check(err, check.IsNil)
 429                 var expectItems []arvados.Collection
 430                 for _, uuid := range trial.expectUUIDs {
 431                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 432                 }
 433                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 434                         Items: expectItems,
 435                 })
 436         }
 437
 438         for i, stub := range s.backends {
 439                 if i >= len(trial.expectCalls) {
 440                         break
 441                 }
 442                 calls := stub.Calls(nil)
 443                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 444                 if len(calls) == 0 {
 445                         continue
 446                 }
 447                 opts := calls[0].Options.(arvados.ListOptions)
 448                 c.Check(opts.Limit, check.Equals, trial.limit)
 449         }
 450 }