lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11         "reflect"
  12         "sort"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 var _ = check.Suite(&CollectionListSuite{})
  20
  21 type collectionLister struct {
  22         arvadostest.APIStub
  23         ItemsToReturn []arvados.Collection
  24         MaxPageSize   int
  25 }
  26
  27 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  28 nextfilter:
  29         for _, f := range filters {
  30                 if f.Attr == "uuid" && f.Operator == "=" {
  31                         s, ok := f.Operand.(string)
  32                         if ok && s == c.UUID {
  33                                 continue nextfilter
  34                         }
  35                 } else if f.Attr == "uuid" && f.Operator == "in" {
  36                         if operand, ok := f.Operand.([]string); ok {
  37                                 for _, s := range operand {
  38                                         if s == c.UUID {
  39                                                 continue nextfilter
  40                                         }
  41                                 }
  42                         } else if operand, ok := f.Operand.([]interface{}); ok {
  43                                 for _, s := range operand {
  44                                         if s, ok := s.(string); ok && s == c.UUID {
  45                                                 continue nextfilter
  46                                         }
  47                                 }
  48                         }
  49                 }
  50                 return false
  51         }
  52         return true
  53 }
  54
  55 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
  56         cl.APIStub.CollectionList(ctx, options)
  57         for _, c := range cl.ItemsToReturn {
  58                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
  59                         break
  60                 }
  61                 if options.Limit >= 0 && int64(len(resp.Items)) >= options.Limit {
  62                         break
  63                 }
  64                 if cl.matchFilters(c, options.Filters) {
  65                         if reflect.DeepEqual(options.Select, []string{"uuid", "name"}) {
  66                                 c = arvados.Collection{UUID: c.UUID, Name: c.Name}
  67                         } else if reflect.DeepEqual(options.Select, []string{"name"}) {
  68                                 c = arvados.Collection{Name: c.Name}
  69                         } else if len(options.Select) > 0 {
  70                                 panic(fmt.Sprintf("not implemented: options=%#v", options))
  71                         }
  72                         resp.Items = append(resp.Items, c)
  73                 }
  74         }
  75         return
  76 }
  77
  78 type CollectionListSuite struct {
  79         FederationSuite
  80         ids      []string   // aaaaa, bbbbb, ccccc
  81         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
  82         backends []*collectionLister
  83 }
  84
  85 func (s *CollectionListSuite) SetUpTest(c *check.C) {
  86         s.FederationSuite.SetUpTest(c)
  87
  88         s.ids = nil
  89         s.uuids = nil
  90         s.backends = nil
  91         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
  92                 cl := &collectionLister{}
  93                 s.ids = append(s.ids, id)
  94                 s.uuids = append(s.uuids, nil)
  95                 for j := 0; j < 5; j++ {
  96                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
  97                         s.uuids[i] = append(s.uuids[i], uuid)
  98                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
  99                                 UUID: uuid,
 100                         })
 101                 }
 102                 s.backends = append(s.backends, cl)
 103                 if i == 0 {
 104                         s.fed.local = cl
 105                 } else if i%1 == 0 {
 106                         // call some backends directly via API
 107                         s.addDirectRemote(c, id, cl)
 108                 } else {
 109                         // call some backends through rpc->router->API
 110                         // to ensure nothing is lost in translation
 111                         s.addHTTPRemote(c, id, cl)
 112                 }
 113         }
 114 }
 115
 116 type listTrial struct {
 117         count        string
 118         limit        int64
 119         offset       int64
 120         order        []string
 121         filters      []arvados.Filter
 122         selectfields []string
 123         expectUUIDs  []string
 124         expectCalls  []int // number of API calls to backends
 125         expectStatus int
 126 }
 127
 128 func (s *CollectionListSuite) TestCollectionListNoUUIDFilters(c *check.C) {
 129         s.test(c, listTrial{
 130                 count:       "none",
 131                 limit:       1,
 132                 expectUUIDs: []string{s.uuids[0][0]},
 133                 expectCalls: []int{1, 0, 0},
 134         })
 135 }
 136
 137 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 138         s.test(c, listTrial{
 139                 count:       "none",
 140                 limit:       -1,
 141                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 142                 expectUUIDs: []string{s.uuids[0][0]},
 143                 expectCalls: []int{1, 0, 0},
 144         })
 145 }
 146
 147 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 148         s.test(c, listTrial{
 149                 count:       "none",
 150                 limit:       -1,
 151                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 152                 expectUUIDs: []string{s.uuids[1][0]},
 153                 expectCalls: []int{0, 1, 0},
 154         })
 155 }
 156
 157 func (s *CollectionListSuite) TestCollectionListOneLocalDeselectingUUID(c *check.C) {
 158         s.test(c, listTrial{
 159                 count:        "none",
 160                 limit:        -1,
 161                 filters:      []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 162                 selectfields: []string{"name"},
 163                 expectUUIDs:  []string{""}, // select=name is honored
 164                 expectCalls:  []int{1, 0, 0},
 165         })
 166 }
 167
 168 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 169         s.test(c, listTrial{
 170                 count:       "none",
 171                 limit:       -1,
 172                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 173                 expectUUIDs: []string{s.uuids[0][0]},
 174                 expectCalls: []int{1, 0, 0},
 175         })
 176 }
 177
 178 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 179         s.test(c, listTrial{
 180                 count:       "none",
 181                 limit:       -1,
 182                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 183                 expectUUIDs: []string{s.uuids[1][1]},
 184                 expectCalls: []int{0, 1, 0},
 185         })
 186 }
 187
 188 func (s *CollectionListSuite) TestCollectionListOneRemoteDeselectingUUID(c *check.C) {
 189         s.test(c, listTrial{
 190                 count:        "none",
 191                 limit:        -1,
 192                 filters:      []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 193                 selectfields: []string{"name"},
 194                 expectUUIDs:  []string{s.uuids[1][0]}, // uuid is returned, despite not being selected
 195                 expectCalls:  []int{0, 1, 0},
 196         })
 197 }
 198
 199 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 200         s.test(c, listTrial{
 201                 count:       "none",
 202                 limit:       -1,
 203                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 204                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 205                 expectCalls: []int{1, 1, 0},
 206         })
 207 }
 208
 209 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemoteDeselectingUUID(c *check.C) {
 210         s.test(c, listTrial{
 211                 count:        "none",
 212                 limit:        -1,
 213                 filters:      []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 214                 selectfields: []string{"name"},
 215                 expectUUIDs:  []string{s.uuids[0][0], s.uuids[1][0]}, // uuid is returned, despite not being selected
 216                 expectCalls:  []int{1, 1, 0},
 217         })
 218 }
 219
 220 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 221         s.test(c, listTrial{
 222                 count:       "none",
 223                 limit:       -1,
 224                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 225                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 226                 expectCalls: []int{0, 1, 1},
 227         })
 228 }
 229
 230 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 231         s.cluster.API.MaxItemsPerResponse = 2
 232         s.test(c, listTrial{
 233                 count: "none",
 234                 limit: -1,
 235                 filters: []arvados.Filter{
 236                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 237                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 238                 },
 239                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 240                 expectCalls: []int{1, 0, 1},
 241         })
 242 }
 243
 244 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 245         s.test(c, listTrial{
 246                 count:       "none",
 247                 limit:       -1,
 248                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 249                 expectUUIDs: []string{},
 250                 expectCalls: []int{0, 0, 0},
 251         })
 252 }
 253
 254 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 255         s.test(c, listTrial{
 256                 count: "none",
 257                 limit: -1,
 258                 filters: []arvados.Filter{
 259                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 260                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 261                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 262                 },
 263                 expectUUIDs: []string{s.uuids[0][0]},
 264                 expectCalls: []int{1, 0, 0},
 265         })
 266 }
 267
 268 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 269         for i := range s.backends {
 270                 s.uuids[i] = s.uuids[i][:3]
 271                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 272         }
 273         s.cluster.API.MaxItemsPerResponse = 9
 274         for _, stub := range s.backends {
 275                 stub.MaxPageSize = 2
 276         }
 277         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 278         s.test(c, listTrial{
 279                 count:       "none",
 280                 limit:       -1,
 281                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 282                 expectUUIDs: allUUIDs,
 283                 expectCalls: []int{2, 2, 2},
 284         })
 285 }
 286
 287 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 288         // not [yet] supported
 289         s.test(c, listTrial{
 290                 count: "none",
 291                 limit: -1,
 292                 filters: []arvados.Filter{
 293                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 294                         {"uuid", "is_a", "teapot"},
 295                 },
 296                 expectCalls:  []int{0, 0, 0},
 297                 expectStatus: http.StatusBadRequest,
 298         })
 299 }
 300
 301 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 302         for _, count := range []string{"", "exact"} {
 303                 s.SetUpTest(c) // Reset backends / call counters
 304                 s.test(c, listTrial{
 305                         count: count,
 306                         limit: -1,
 307                         filters: []arvados.Filter{
 308                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 309                                 {"uuid", "is_a", "teapot"},
 310                         },
 311                         expectCalls:  []int{0, 0, 0},
 312                         expectStatus: http.StatusBadRequest,
 313                 })
 314         }
 315 }
 316
 317 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 318         for _, limit := range []int64{0, 1, 2} {
 319                 s.SetUpTest(c) // Reset backends / call counters
 320                 s.test(c, listTrial{
 321                         count: "none",
 322                         limit: limit,
 323                         filters: []arvados.Filter{
 324                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0], s.uuids[2][0]}},
 325                                 {"uuid", "is_a", "teapot"},
 326                         },
 327                         expectCalls:  []int{0, 0, 0},
 328                         expectStatus: http.StatusBadRequest,
 329                 })
 330         }
 331 }
 332
 333 func (s *CollectionListSuite) TestCollectionListMultiSiteWithHighLimit(c *check.C) {
 334         uuids := []string{s.uuids[0][0], s.uuids[1][0], s.uuids[2][0]}
 335         for _, limit := range []int64{3, 4, 1234567890} {
 336                 s.SetUpTest(c) // Reset backends / call counters
 337                 s.test(c, listTrial{
 338                         count: "none",
 339                         limit: limit,
 340                         filters: []arvados.Filter{
 341                                 {"uuid", "in", uuids},
 342                         },
 343                         expectUUIDs: uuids,
 344                         expectCalls: []int{1, 1, 1},
 345                 })
 346         }
 347 }
 348
 349 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 350         s.test(c, listTrial{
 351                 count:  "none",
 352                 limit:  -1,
 353                 offset: 1,
 354                 filters: []arvados.Filter{
 355                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 356                         {"uuid", "is_a", "teapot"},
 357                 },
 358                 expectCalls:  []int{0, 0, 0},
 359                 expectStatus: http.StatusBadRequest,
 360         })
 361 }
 362
 363 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 364         s.test(c, listTrial{
 365                 count: "none",
 366                 limit: -1,
 367                 order: []string{"uuid desc"},
 368                 filters: []arvados.Filter{
 369                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 370                         {"uuid", "is_a", "teapot"},
 371                 },
 372                 expectCalls:  []int{0, 0, 0},
 373                 expectStatus: http.StatusBadRequest,
 374         })
 375 }
 376
 377 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 378         s.test(c, listTrial{
 379                 count: "none",
 380                 limit: -1,
 381                 filters: []arvados.Filter{
 382                         {"uuid", "in", "teapot"},
 383                 },
 384                 expectCalls:  []int{0, 0, 0},
 385                 expectStatus: http.StatusBadRequest,
 386         })
 387 }
 388
 389 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 390         s.test(c, listTrial{
 391                 count: "none",
 392                 limit: -1,
 393                 filters: []arvados.Filter{
 394                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 395                 },
 396                 expectStatus: http.StatusNotFound,
 397         })
 398 }
 399
 400 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 401         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{Error: fmt.Errorf("stub backend error")})
 402         s.test(c, listTrial{
 403                 count: "none",
 404                 limit: -1,
 405                 filters: []arvados.Filter{
 406                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 407                 },
 408                 expectStatus: http.StatusBadGateway,
 409         })
 410 }
 411
 412 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 413         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 414                 Count:   trial.count,
 415                 Limit:   trial.limit,
 416                 Offset:  trial.offset,
 417                 Order:   trial.order,
 418                 Filters: trial.filters,
 419                 Select:  trial.selectfields,
 420         })
 421         if trial.expectStatus != 0 {
 422                 c.Assert(err, check.NotNil)
 423                 err, ok := err.(interface{ HTTPStatus() int })
 424                 c.Assert(ok, check.Equals, true) // err must implement interface{ HTTPStatus() int }
 425                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 426                 c.Logf("returned error is %#v", err)
 427                 c.Logf("returned error string is %q", err)
 428         } else {
 429                 c.Check(err, check.IsNil)
 430                 expectItems := []arvados.Collection{}
 431                 for _, uuid := range trial.expectUUIDs {
 432                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 433                 }
 434                 // expectItems is sorted by UUID, so sort resp.Items
 435                 // by UUID before checking DeepEquals.
 436                 sort.Slice(resp.Items, func(i, j int) bool { return resp.Items[i].UUID < resp.Items[j].UUID })
 437                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 438                         Items: expectItems,
 439                 })
 440         }
 441
 442         for i, stub := range s.backends {
 443                 if i >= len(trial.expectCalls) {
 444                         break
 445                 }
 446                 calls := stub.Calls(nil)
 447                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 448                 if len(calls) == 0 {
 449                         continue
 450                 }
 451                 opts := calls[0].Options.(arvados.ListOptions)
 452                 c.Check(opts.Limit, check.Equals, trial.limit)
 453         }
 454 }