lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvados"
  13         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  14         check "gopkg.in/check.v1"
  15 )
  16
  17 var _ = check.Suite(&CollectionListSuite{})
  18
  19 type collectionLister struct {
  20         arvadostest.APIStub
  21         ItemsToReturn []arvados.Collection
  22         MaxPageSize   int
  23 }
  24
  25 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  26 nextfilter:
  27         for _, f := range filters {
  28                 if f.Attr == "uuid" && f.Operator == "=" {
  29                         s, ok := f.Operand.(string)
  30                         if ok && s == c.UUID {
  31                                 continue nextfilter
  32                         }
  33                 } else if f.Attr == "uuid" && f.Operator == "in" {
  34                         if operand, ok := f.Operand.([]string); ok {
  35                                 for _, s := range operand {
  36                                         if s == c.UUID {
  37                                                 continue nextfilter
  38                                         }
  39                                 }
  40                         } else if operand, ok := f.Operand.([]interface{}); ok {
  41                                 for _, s := range operand {
  42                                         if s, ok := s.(string); ok && s == c.UUID {
  43                                                 continue nextfilter
  44                                         }
  45                                 }
  46                         }
  47                 }
  48                 return false
  49         }
  50         return true
  51 }
  52
  53 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
  54         cl.APIStub.CollectionList(ctx, options)
  55         for _, c := range cl.ItemsToReturn {
  56                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
  57                         break
  58                 }
  59                 if options.Limit >= 0 && len(resp.Items) >= options.Limit {
  60                         break
  61                 }
  62                 if cl.matchFilters(c, options.Filters) {
  63                         resp.Items = append(resp.Items, c)
  64                 }
  65         }
  66         return
  67 }
  68
  69 type CollectionListSuite struct {
  70         FederationSuite
  71         ids      []string   // aaaaa, bbbbb, ccccc
  72         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
  73         backends []*collectionLister
  74 }
  75
  76 func (s *CollectionListSuite) SetUpTest(c *check.C) {
  77         s.FederationSuite.SetUpTest(c)
  78
  79         s.ids = nil
  80         s.uuids = nil
  81         s.backends = nil
  82         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
  83                 cl := &collectionLister{}
  84                 s.ids = append(s.ids, id)
  85                 s.uuids = append(s.uuids, nil)
  86                 for j := 0; j < 5; j++ {
  87                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
  88                         s.uuids[i] = append(s.uuids[i], uuid)
  89                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
  90                                 UUID: uuid,
  91                         })
  92                 }
  93                 s.backends = append(s.backends, cl)
  94                 if i == 0 {
  95                         s.fed.local = cl
  96                 } else if i%1 == 0 {
  97                         // call some backends directly via API
  98                         s.addDirectRemote(c, id, cl)
  99                 } else {
 100                         // call some backends through rpc->router->API
 101                         // to ensure nothing is lost in translation
 102                         s.addHTTPRemote(c, id, cl)
 103                 }
 104         }
 105 }
 106
 107 type listTrial struct {
 108         count        string
 109         limit        int
 110         offset       int
 111         order        []string
 112         filters      []arvados.Filter
 113         expectUUIDs  []string
 114         expectCalls  []int // number of API calls to backends
 115         expectStatus int
 116 }
 117
 118 func (s *CollectionListSuite) TestCollectionListNoUUIDFilters(c *check.C) {
 119         s.test(c, listTrial{
 120                 count:       "none",
 121                 limit:       1,
 122                 expectUUIDs: []string{s.uuids[0][0]},
 123                 expectCalls: []int{1, 0, 0},
 124         })
 125 }
 126
 127 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 128         s.test(c, listTrial{
 129                 count:       "none",
 130                 limit:       -1,
 131                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 132                 expectUUIDs: []string{s.uuids[0][0]},
 133                 expectCalls: []int{1, 0, 0},
 134         })
 135 }
 136
 137 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 138         s.test(c, listTrial{
 139                 count:       "none",
 140                 limit:       -1,
 141                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 142                 expectUUIDs: []string{s.uuids[1][0]},
 143                 expectCalls: []int{0, 1, 0},
 144         })
 145 }
 146
 147 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 148         s.test(c, listTrial{
 149                 count:       "none",
 150                 limit:       -1,
 151                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 152                 expectUUIDs: []string{s.uuids[0][0]},
 153                 expectCalls: []int{1, 0, 0},
 154         })
 155 }
 156
 157 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 158         s.test(c, listTrial{
 159                 count:       "none",
 160                 limit:       -1,
 161                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 162                 expectUUIDs: []string{s.uuids[1][1]},
 163                 expectCalls: []int{0, 1, 0},
 164         })
 165 }
 166
 167 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 168         s.test(c, listTrial{
 169                 count:       "none",
 170                 limit:       -1,
 171                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 172                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 173                 expectCalls: []int{1, 1, 0},
 174         })
 175 }
 176
 177 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 178         s.test(c, listTrial{
 179                 count:       "none",
 180                 limit:       -1,
 181                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 182                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 183                 expectCalls: []int{0, 1, 1},
 184         })
 185 }
 186
 187 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 188         s.cluster.API.MaxItemsPerResponse = 2
 189         s.test(c, listTrial{
 190                 count: "none",
 191                 limit: -1,
 192                 filters: []arvados.Filter{
 193                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 194                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 195                 },
 196                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 197                 expectCalls: []int{1, 0, 1},
 198         })
 199 }
 200
 201 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 202         s.test(c, listTrial{
 203                 count:       "none",
 204                 limit:       -1,
 205                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 206                 expectUUIDs: []string{},
 207                 expectCalls: []int{0, 0, 0},
 208         })
 209 }
 210
 211 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 212         s.test(c, listTrial{
 213                 count: "none",
 214                 limit: -1,
 215                 filters: []arvados.Filter{
 216                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 217                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 218                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 219                 },
 220                 expectUUIDs: []string{s.uuids[0][0]},
 221                 expectCalls: []int{1, 0, 0},
 222         })
 223 }
 224
 225 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 226         for i := range s.backends {
 227                 s.uuids[i] = s.uuids[i][:3]
 228                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 229         }
 230         s.cluster.API.MaxItemsPerResponse = 9
 231         for _, stub := range s.backends {
 232                 stub.MaxPageSize = 2
 233         }
 234         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 235         s.test(c, listTrial{
 236                 count:       "none",
 237                 limit:       -1,
 238                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 239                 expectUUIDs: allUUIDs,
 240                 expectCalls: []int{2, 2, 2},
 241         })
 242 }
 243
 244 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 245         // not [yet] supported
 246         s.test(c, listTrial{
 247                 count: "none",
 248                 limit: -1,
 249                 filters: []arvados.Filter{
 250                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 251                         {"uuid", "is_a", "teapot"},
 252                 },
 253                 expectCalls:  []int{0, 0, 0},
 254                 expectStatus: http.StatusBadRequest,
 255         })
 256 }
 257
 258 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 259         for _, count := range []string{"", "exact"} {
 260                 s.test(c, listTrial{
 261                         count: count,
 262                         limit: -1,
 263                         filters: []arvados.Filter{
 264                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 265                                 {"uuid", "is_a", "teapot"},
 266                         },
 267                         expectCalls:  []int{0, 0, 0},
 268                         expectStatus: http.StatusBadRequest,
 269                 })
 270         }
 271 }
 272
 273 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 274         for _, limit := range []int{0, 1, 2} {
 275                 s.test(c, listTrial{
 276                         count: "none",
 277                         limit: limit,
 278                         filters: []arvados.Filter{
 279                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 280                                 {"uuid", "is_a", "teapot"},
 281                         },
 282                         expectCalls:  []int{0, 0, 0},
 283                         expectStatus: http.StatusBadRequest,
 284                 })
 285         }
 286 }
 287
 288 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 289         s.test(c, listTrial{
 290                 count:  "none",
 291                 limit:  -1,
 292                 offset: 1,
 293                 filters: []arvados.Filter{
 294                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 295                         {"uuid", "is_a", "teapot"},
 296                 },
 297                 expectCalls:  []int{0, 0, 0},
 298                 expectStatus: http.StatusBadRequest,
 299         })
 300 }
 301
 302 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 303         s.test(c, listTrial{
 304                 count: "none",
 305                 limit: -1,
 306                 order: []string{"uuid desc"},
 307                 filters: []arvados.Filter{
 308                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 309                         {"uuid", "is_a", "teapot"},
 310                 },
 311                 expectCalls:  []int{0, 0, 0},
 312                 expectStatus: http.StatusBadRequest,
 313         })
 314 }
 315
 316 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 317         s.test(c, listTrial{
 318                 count: "none",
 319                 limit: -1,
 320                 filters: []arvados.Filter{
 321                         {"uuid", "in", "teapot"},
 322                 },
 323                 expectCalls:  []int{0, 0, 0},
 324                 expectStatus: http.StatusBadRequest,
 325         })
 326 }
 327
 328 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 329         s.test(c, listTrial{
 330                 count: "none",
 331                 limit: -1,
 332                 filters: []arvados.Filter{
 333                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 334                 },
 335                 expectStatus: http.StatusNotFound,
 336         })
 337 }
 338
 339 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 340         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{})
 341         s.test(c, listTrial{
 342                 count: "none",
 343                 limit: -1,
 344                 filters: []arvados.Filter{
 345                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 346                 },
 347                 expectStatus: http.StatusBadGateway,
 348         })
 349 }
 350
 351 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 352         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 353                 Count:   trial.count,
 354                 Limit:   trial.limit,
 355                 Offset:  trial.offset,
 356                 Order:   trial.order,
 357                 Filters: trial.filters,
 358         })
 359         if trial.expectStatus != 0 {
 360                 c.Assert(err, check.NotNil)
 361                 err, _ := err.(interface{ HTTPStatus() int })
 362                 c.Assert(err, check.NotNil) // err must implement HTTPStatus()
 363                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 364                 c.Logf("returned error is %#v", err)
 365                 c.Logf("returned error string is %q", err)
 366         } else {
 367                 c.Check(err, check.IsNil)
 368                 var expectItems []arvados.Collection
 369                 for _, uuid := range trial.expectUUIDs {
 370                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 371                 }
 372                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 373                         Items: expectItems,
 374                 })
 375         }
 376
 377         for i, stub := range s.backends {
 378                 if i >= len(trial.expectCalls) {
 379                         break
 380                 }
 381                 calls := stub.Calls(nil)
 382                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 383                 if len(calls) == 0 {
 384                         continue
 385                 }
 386                 opts := calls[0].Options.(arvados.ListOptions)
 387                 c.Check(opts.Limit, check.Equals, trial.limit)
 388         }
 389 }