lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11         "sort"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/arvados"
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 var _ = check.Suite(&CollectionListSuite{})
  19
  20 type collectionLister struct {
  21         arvadostest.APIStub
  22         ItemsToReturn []arvados.Collection
  23         MaxPageSize   int
  24 }
  25
  26 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  27 nextfilter:
  28         for _, f := range filters {
  29                 if f.Attr == "uuid" && f.Operator == "=" {
  30                         s, ok := f.Operand.(string)
  31                         if ok && s == c.UUID {
  32                                 continue nextfilter
  33                         }
  34                 } else if f.Attr == "uuid" && f.Operator == "in" {
  35                         if operand, ok := f.Operand.([]string); ok {
  36                                 for _, s := range operand {
  37                                         if s == c.UUID {
  38                                                 continue nextfilter
  39                                         }
  40                                 }
  41                         } else if operand, ok := f.Operand.([]interface{}); ok {
  42                                 for _, s := range operand {
  43                                         if s, ok := s.(string); ok && s == c.UUID {
  44                                                 continue nextfilter
  45                                         }
  46                                 }
  47                         }
  48                 }
  49                 return false
  50         }
  51         return true
  52 }
  53
  54 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
  55         cl.APIStub.CollectionList(ctx, options)
  56         for _, c := range cl.ItemsToReturn {
  57                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
  58                         break
  59                 }
  60                 if options.Limit >= 0 && len(resp.Items) >= options.Limit {
  61                         break
  62                 }
  63                 if cl.matchFilters(c, options.Filters) {
  64                         resp.Items = append(resp.Items, c)
  65                 }
  66         }
  67         return
  68 }
  69
  70 type CollectionListSuite struct {
  71         FederationSuite
  72         ids      []string   // aaaaa, bbbbb, ccccc
  73         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
  74         backends []*collectionLister
  75 }
  76
  77 func (s *CollectionListSuite) SetUpTest(c *check.C) {
  78         s.FederationSuite.SetUpTest(c)
  79
  80         s.ids = nil
  81         s.uuids = nil
  82         s.backends = nil
  83         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
  84                 cl := &collectionLister{}
  85                 s.ids = append(s.ids, id)
  86                 s.uuids = append(s.uuids, nil)
  87                 for j := 0; j < 5; j++ {
  88                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
  89                         s.uuids[i] = append(s.uuids[i], uuid)
  90                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
  91                                 UUID: uuid,
  92                         })
  93                 }
  94                 s.backends = append(s.backends, cl)
  95                 if i == 0 {
  96                         s.fed.local = cl
  97                 } else if i%1 == 0 {
  98                         // call some backends directly via API
  99                         s.addDirectRemote(c, id, cl)
 100                 } else {
 101                         // call some backends through rpc->router->API
 102                         // to ensure nothing is lost in translation
 103                         s.addHTTPRemote(c, id, cl)
 104                 }
 105         }
 106 }
 107
 108 type listTrial struct {
 109         count        string
 110         limit        int
 111         offset       int
 112         order        []string
 113         filters      []arvados.Filter
 114         expectUUIDs  []string
 115         expectCalls  []int // number of API calls to backends
 116         expectStatus int
 117 }
 118
 119 func (s *CollectionListSuite) TestCollectionListNoUUIDFilters(c *check.C) {
 120         s.test(c, listTrial{
 121                 count:       "none",
 122                 limit:       1,
 123                 expectUUIDs: []string{s.uuids[0][0]},
 124                 expectCalls: []int{1, 0, 0},
 125         })
 126 }
 127
 128 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 129         s.test(c, listTrial{
 130                 count:       "none",
 131                 limit:       -1,
 132                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 133                 expectUUIDs: []string{s.uuids[0][0]},
 134                 expectCalls: []int{1, 0, 0},
 135         })
 136 }
 137
 138 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 139         s.test(c, listTrial{
 140                 count:       "none",
 141                 limit:       -1,
 142                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 143                 expectUUIDs: []string{s.uuids[1][0]},
 144                 expectCalls: []int{0, 1, 0},
 145         })
 146 }
 147
 148 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 149         s.test(c, listTrial{
 150                 count:       "none",
 151                 limit:       -1,
 152                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 153                 expectUUIDs: []string{s.uuids[0][0]},
 154                 expectCalls: []int{1, 0, 0},
 155         })
 156 }
 157
 158 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 159         s.test(c, listTrial{
 160                 count:       "none",
 161                 limit:       -1,
 162                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 163                 expectUUIDs: []string{s.uuids[1][1]},
 164                 expectCalls: []int{0, 1, 0},
 165         })
 166 }
 167
 168 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 169         s.test(c, listTrial{
 170                 count:       "none",
 171                 limit:       -1,
 172                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 173                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 174                 expectCalls: []int{1, 1, 0},
 175         })
 176 }
 177
 178 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 179         s.test(c, listTrial{
 180                 count:       "none",
 181                 limit:       -1,
 182                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 183                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 184                 expectCalls: []int{0, 1, 1},
 185         })
 186 }
 187
 188 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 189         s.cluster.API.MaxItemsPerResponse = 2
 190         s.test(c, listTrial{
 191                 count: "none",
 192                 limit: -1,
 193                 filters: []arvados.Filter{
 194                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 195                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 196                 },
 197                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 198                 expectCalls: []int{1, 0, 1},
 199         })
 200 }
 201
 202 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 203         s.test(c, listTrial{
 204                 count:       "none",
 205                 limit:       -1,
 206                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 207                 expectUUIDs: []string{},
 208                 expectCalls: []int{0, 0, 0},
 209         })
 210 }
 211
 212 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 213         s.test(c, listTrial{
 214                 count: "none",
 215                 limit: -1,
 216                 filters: []arvados.Filter{
 217                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 218                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 219                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 220                 },
 221                 expectUUIDs: []string{s.uuids[0][0]},
 222                 expectCalls: []int{1, 0, 0},
 223         })
 224 }
 225
 226 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 227         for i := range s.backends {
 228                 s.uuids[i] = s.uuids[i][:3]
 229                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 230         }
 231         s.cluster.API.MaxItemsPerResponse = 9
 232         for _, stub := range s.backends {
 233                 stub.MaxPageSize = 2
 234         }
 235         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 236         s.test(c, listTrial{
 237                 count:       "none",
 238                 limit:       -1,
 239                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 240                 expectUUIDs: allUUIDs,
 241                 expectCalls: []int{2, 2, 2},
 242         })
 243 }
 244
 245 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 246         // not [yet] supported
 247         s.test(c, listTrial{
 248                 count: "none",
 249                 limit: -1,
 250                 filters: []arvados.Filter{
 251                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 252                         {"uuid", "is_a", "teapot"},
 253                 },
 254                 expectCalls:  []int{0, 0, 0},
 255                 expectStatus: http.StatusBadRequest,
 256         })
 257 }
 258
 259 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 260         for _, count := range []string{"", "exact"} {
 261                 s.test(c, listTrial{
 262                         count: count,
 263                         limit: -1,
 264                         filters: []arvados.Filter{
 265                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 266                                 {"uuid", "is_a", "teapot"},
 267                         },
 268                         expectCalls:  []int{0, 0, 0},
 269                         expectStatus: http.StatusBadRequest,
 270                 })
 271         }
 272 }
 273
 274 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 275         for _, limit := range []int{0, 1, 2} {
 276                 s.test(c, listTrial{
 277                         count: "none",
 278                         limit: limit,
 279                         filters: []arvados.Filter{
 280                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 281                                 {"uuid", "is_a", "teapot"},
 282                         },
 283                         expectCalls:  []int{0, 0, 0},
 284                         expectStatus: http.StatusBadRequest,
 285                 })
 286         }
 287 }
 288
 289 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 290         s.test(c, listTrial{
 291                 count:  "none",
 292                 limit:  -1,
 293                 offset: 1,
 294                 filters: []arvados.Filter{
 295                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 296                         {"uuid", "is_a", "teapot"},
 297                 },
 298                 expectCalls:  []int{0, 0, 0},
 299                 expectStatus: http.StatusBadRequest,
 300         })
 301 }
 302
 303 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 304         s.test(c, listTrial{
 305                 count: "none",
 306                 limit: -1,
 307                 order: []string{"uuid desc"},
 308                 filters: []arvados.Filter{
 309                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 310                         {"uuid", "is_a", "teapot"},
 311                 },
 312                 expectCalls:  []int{0, 0, 0},
 313                 expectStatus: http.StatusBadRequest,
 314         })
 315 }
 316
 317 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 318         s.test(c, listTrial{
 319                 count: "none",
 320                 limit: -1,
 321                 filters: []arvados.Filter{
 322                         {"uuid", "in", "teapot"},
 323                 },
 324                 expectCalls:  []int{0, 0, 0},
 325                 expectStatus: http.StatusBadRequest,
 326         })
 327 }
 328
 329 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 330         s.test(c, listTrial{
 331                 count: "none",
 332                 limit: -1,
 333                 filters: []arvados.Filter{
 334                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 335                 },
 336                 expectStatus: http.StatusNotFound,
 337         })
 338 }
 339
 340 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 341         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{})
 342         s.test(c, listTrial{
 343                 count: "none",
 344                 limit: -1,
 345                 filters: []arvados.Filter{
 346                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 347                 },
 348                 expectStatus: http.StatusBadGateway,
 349         })
 350 }
 351
 352 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 353         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 354                 Count:   trial.count,
 355                 Limit:   trial.limit,
 356                 Offset:  trial.offset,
 357                 Order:   trial.order,
 358                 Filters: trial.filters,
 359         })
 360         if trial.expectStatus != 0 {
 361                 c.Assert(err, check.NotNil)
 362                 err, _ := err.(interface{ HTTPStatus() int })
 363                 c.Assert(err, check.NotNil) // err must implement HTTPStatus()
 364                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 365                 c.Logf("returned error is %#v", err)
 366                 c.Logf("returned error string is %q", err)
 367         } else {
 368                 c.Check(err, check.IsNil)
 369                 expectItems := []arvados.Collection{}
 370                 for _, uuid := range trial.expectUUIDs {
 371                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 372                 }
 373                 // expectItems is sorted by UUID, so sort resp.Items
 374                 // by UUID before checking DeepEquals.
 375                 sort.Slice(resp.Items, func(i, j int) bool { return resp.Items[i].UUID < resp.Items[j].UUID })
 376                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 377                         Items: expectItems,
 378                 })
 379         }
 380
 381         for i, stub := range s.backends {
 382                 if i >= len(trial.expectCalls) {
 383                         break
 384                 }
 385                 calls := stub.Calls(nil)
 386                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 387                 if len(calls) == 0 {
 388                         continue
 389                 }
 390                 opts := calls[0].Options.(arvados.ListOptions)
 391                 c.Check(opts.Limit, check.Equals, trial.limit)
 392         }
 393 }