lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11         "reflect"
  12         "sort"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 var _ = check.Suite(&CollectionListSuite{})
  20
  21 type collectionLister struct {
  22         arvadostest.APIStub
  23         ItemsToReturn []arvados.Collection
  24         MaxPageSize   int
  25 }
  26
  27 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  28 nextfilter:
  29         for _, f := range filters {
  30                 if f.Attr == "uuid" && f.Operator == "=" {
  31                         s, ok := f.Operand.(string)
  32                         if ok && s == c.UUID {
  33                                 continue nextfilter
  34                         }
  35                 } else if f.Attr == "uuid" && f.Operator == "in" {
  36                         if operand, ok := f.Operand.([]string); ok {
  37                                 for _, s := range operand {
  38                                         if s == c.UUID {
  39                                                 continue nextfilter
  40                                         }
  41                                 }
  42                         } else if operand, ok := f.Operand.([]interface{}); ok {
  43                                 for _, s := range operand {
  44                                         if s, ok := s.(string); ok && s == c.UUID {
  45                                                 continue nextfilter
  46                                         }
  47                                 }
  48                         }
  49                 }
  50                 return false
  51         }
  52         return true
  53 }
  54
  55 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
  56         cl.APIStub.CollectionList(ctx, options)
  57         for _, c := range cl.ItemsToReturn {
  58                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
  59                         break
  60                 }
  61                 if options.Limit >= 0 && int64(len(resp.Items)) >= options.Limit {
  62                         break
  63                 }
  64                 if cl.matchFilters(c, options.Filters) {
  65                         if reflect.DeepEqual(options.Select, []string{"uuid", "name"}) {
  66                                 c = arvados.Collection{UUID: c.UUID, Name: c.Name}
  67                         } else if reflect.DeepEqual(options.Select, []string{"name"}) {
  68                                 c = arvados.Collection{Name: c.Name}
  69                         } else if len(options.Select) > 0 {
  70                                 panic(fmt.Sprintf("not implemented: options=%#v", options))
  71                         }
  72                         resp.Items = append(resp.Items, c)
  73                 }
  74         }
  75         return
  76 }
  77
  78 type CollectionListSuite struct {
  79         FederationSuite
  80         ids      []string   // aaaaa, bbbbb, ccccc
  81         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
  82         backends []*collectionLister
  83 }
  84
  85 func (s *CollectionListSuite) SetUpTest(c *check.C) {
  86         s.FederationSuite.SetUpTest(c)
  87
  88         s.ids = nil
  89         s.uuids = nil
  90         s.backends = nil
  91         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
  92                 cl := &collectionLister{}
  93                 s.ids = append(s.ids, id)
  94                 s.uuids = append(s.uuids, nil)
  95                 for j := 0; j < 5; j++ {
  96                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
  97                         s.uuids[i] = append(s.uuids[i], uuid)
  98                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
  99                                 UUID: uuid,
 100                         })
 101                 }
 102                 s.backends = append(s.backends, cl)
 103                 if i == 0 {
 104                         s.fed.local = cl
 105                 } else if i%1 == 0 {
 106                         // call some backends directly via API
 107                         s.addDirectRemote(c, id, cl)
 108                 } else {
 109                         // call some backends through rpc->router->API
 110                         // to ensure nothing is lost in translation
 111                         s.addHTTPRemote(c, id, cl)
 112                 }
 113         }
 114 }
 115
 116 type listTrial struct {
 117         count        string
 118         limit        int64
 119         offset       int64
 120         order        []string
 121         filters      []arvados.Filter
 122         selectfields []string
 123         expectUUIDs  []string
 124         expectCalls  []int // number of API calls to backends
 125         expectStatus int
 126 }
 127
 128 func (s *CollectionListSuite) TestCollectionListNoUUIDFilters(c *check.C) {
 129         s.test(c, listTrial{
 130                 count:       "none",
 131                 limit:       1,
 132                 expectUUIDs: []string{s.uuids[0][0]},
 133                 expectCalls: []int{1, 0, 0},
 134         })
 135 }
 136
 137 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 138         s.test(c, listTrial{
 139                 count:       "none",
 140                 limit:       -1,
 141                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 142                 expectUUIDs: []string{s.uuids[0][0]},
 143                 expectCalls: []int{1, 0, 0},
 144         })
 145 }
 146
 147 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 148         s.test(c, listTrial{
 149                 count:       "none",
 150                 limit:       -1,
 151                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 152                 expectUUIDs: []string{s.uuids[1][0]},
 153                 expectCalls: []int{0, 1, 0},
 154         })
 155 }
 156
 157 func (s *CollectionListSuite) TestCollectionListOneLocalDeselectingUUID(c *check.C) {
 158         s.test(c, listTrial{
 159                 count:        "none",
 160                 limit:        -1,
 161                 filters:      []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 162                 selectfields: []string{"name"},
 163                 expectUUIDs:  []string{""}, // select=name is honored
 164                 expectCalls:  []int{1, 0, 0},
 165         })
 166 }
 167
 168 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 169         s.test(c, listTrial{
 170                 count:       "none",
 171                 limit:       -1,
 172                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 173                 expectUUIDs: []string{s.uuids[0][0]},
 174                 expectCalls: []int{1, 0, 0},
 175         })
 176 }
 177
 178 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 179         s.test(c, listTrial{
 180                 count:       "none",
 181                 limit:       -1,
 182                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 183                 expectUUIDs: []string{s.uuids[1][1]},
 184                 expectCalls: []int{0, 1, 0},
 185         })
 186 }
 187
 188 func (s *CollectionListSuite) TestCollectionListOneRemoteDeselectingUUID(c *check.C) {
 189         s.test(c, listTrial{
 190                 count:        "none",
 191                 limit:        -1,
 192                 filters:      []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 193                 selectfields: []string{"name"},
 194                 expectUUIDs:  []string{s.uuids[1][0]}, // uuid is returned, despite not being selected
 195                 expectCalls:  []int{0, 1, 0},
 196         })
 197 }
 198
 199 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 200         s.test(c, listTrial{
 201                 count:       "none",
 202                 limit:       -1,
 203                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 204                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 205                 expectCalls: []int{1, 1, 0},
 206         })
 207 }
 208
 209 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemoteDeselectingUUID(c *check.C) {
 210         s.test(c, listTrial{
 211                 count:        "none",
 212                 limit:        -1,
 213                 filters:      []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 214                 selectfields: []string{"name"},
 215                 expectUUIDs:  []string{s.uuids[0][0], s.uuids[1][0]}, // uuid is returned, despite not being selected
 216                 expectCalls:  []int{1, 1, 0},
 217         })
 218 }
 219
 220 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 221         s.test(c, listTrial{
 222                 count:       "none",
 223                 limit:       -1,
 224                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 225                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 226                 expectCalls: []int{0, 1, 1},
 227         })
 228 }
 229
 230 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 231         s.cluster.API.MaxItemsPerResponse = 2
 232         s.test(c, listTrial{
 233                 count: "none",
 234                 limit: -1,
 235                 filters: []arvados.Filter{
 236                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 237                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 238                 },
 239                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 240                 expectCalls: []int{1, 0, 1},
 241         })
 242 }
 243
 244 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 245         s.test(c, listTrial{
 246                 count:       "none",
 247                 limit:       -1,
 248                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 249                 expectUUIDs: []string{},
 250                 expectCalls: []int{0, 0, 0},
 251         })
 252 }
 253
 254 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 255         s.test(c, listTrial{
 256                 count: "none",
 257                 limit: -1,
 258                 filters: []arvados.Filter{
 259                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 260                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 261                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 262                 },
 263                 expectUUIDs: []string{s.uuids[0][0]},
 264                 expectCalls: []int{1, 0, 0},
 265         })
 266 }
 267
 268 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 269         for i := range s.backends {
 270                 s.uuids[i] = s.uuids[i][:3]
 271                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 272         }
 273         s.cluster.API.MaxItemsPerResponse = 9
 274         for _, stub := range s.backends {
 275                 stub.MaxPageSize = 2
 276         }
 277         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 278         s.test(c, listTrial{
 279                 count:       "none",
 280                 limit:       -1,
 281                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 282                 expectUUIDs: allUUIDs,
 283                 expectCalls: []int{2, 2, 2},
 284         })
 285 }
 286
 287 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 288         // not [yet] supported
 289         s.test(c, listTrial{
 290                 count: "none",
 291                 limit: -1,
 292                 filters: []arvados.Filter{
 293                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 294                         {"uuid", "is_a", "teapot"},
 295                 },
 296                 expectCalls:  []int{0, 0, 0},
 297                 expectStatus: http.StatusBadRequest,
 298         })
 299 }
 300
 301 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 302         for _, count := range []string{"", "exact"} {
 303                 s.test(c, listTrial{
 304                         count: count,
 305                         limit: -1,
 306                         filters: []arvados.Filter{
 307                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 308                                 {"uuid", "is_a", "teapot"},
 309                         },
 310                         expectCalls:  []int{0, 0, 0},
 311                         expectStatus: http.StatusBadRequest,
 312                 })
 313         }
 314 }
 315
 316 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 317         for _, limit := range []int64{0, 1, 2} {
 318                 s.test(c, listTrial{
 319                         count: "none",
 320                         limit: limit,
 321                         filters: []arvados.Filter{
 322                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 323                                 {"uuid", "is_a", "teapot"},
 324                         },
 325                         expectCalls:  []int{0, 0, 0},
 326                         expectStatus: http.StatusBadRequest,
 327                 })
 328         }
 329 }
 330
 331 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 332         s.test(c, listTrial{
 333                 count:  "none",
 334                 limit:  -1,
 335                 offset: 1,
 336                 filters: []arvados.Filter{
 337                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 338                         {"uuid", "is_a", "teapot"},
 339                 },
 340                 expectCalls:  []int{0, 0, 0},
 341                 expectStatus: http.StatusBadRequest,
 342         })
 343 }
 344
 345 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 346         s.test(c, listTrial{
 347                 count: "none",
 348                 limit: -1,
 349                 order: []string{"uuid desc"},
 350                 filters: []arvados.Filter{
 351                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 352                         {"uuid", "is_a", "teapot"},
 353                 },
 354                 expectCalls:  []int{0, 0, 0},
 355                 expectStatus: http.StatusBadRequest,
 356         })
 357 }
 358
 359 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 360         s.test(c, listTrial{
 361                 count: "none",
 362                 limit: -1,
 363                 filters: []arvados.Filter{
 364                         {"uuid", "in", "teapot"},
 365                 },
 366                 expectCalls:  []int{0, 0, 0},
 367                 expectStatus: http.StatusBadRequest,
 368         })
 369 }
 370
 371 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 372         s.test(c, listTrial{
 373                 count: "none",
 374                 limit: -1,
 375                 filters: []arvados.Filter{
 376                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 377                 },
 378                 expectStatus: http.StatusNotFound,
 379         })
 380 }
 381
 382 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 383         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{Error: fmt.Errorf("stub backend error")})
 384         s.test(c, listTrial{
 385                 count: "none",
 386                 limit: -1,
 387                 filters: []arvados.Filter{
 388                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 389                 },
 390                 expectStatus: http.StatusBadGateway,
 391         })
 392 }
 393
 394 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 395         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 396                 Count:   trial.count,
 397                 Limit:   trial.limit,
 398                 Offset:  trial.offset,
 399                 Order:   trial.order,
 400                 Filters: trial.filters,
 401                 Select:  trial.selectfields,
 402         })
 403         if trial.expectStatus != 0 {
 404                 c.Assert(err, check.NotNil)
 405                 err, ok := err.(interface{ HTTPStatus() int })
 406                 c.Assert(ok, check.Equals, true) // err must implement interface{ HTTPStatus() int }
 407                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 408                 c.Logf("returned error is %#v", err)
 409                 c.Logf("returned error string is %q", err)
 410         } else {
 411                 c.Check(err, check.IsNil)
 412                 expectItems := []arvados.Collection{}
 413                 for _, uuid := range trial.expectUUIDs {
 414                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 415                 }
 416                 // expectItems is sorted by UUID, so sort resp.Items
 417                 // by UUID before checking DeepEquals.
 418                 sort.Slice(resp.Items, func(i, j int) bool { return resp.Items[i].UUID < resp.Items[j].UUID })
 419                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 420                         Items: expectItems,
 421                 })
 422         }
 423
 424         for i, stub := range s.backends {
 425                 if i >= len(trial.expectCalls) {
 426                         break
 427                 }
 428                 calls := stub.Calls(nil)
 429                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 430                 if len(calls) == 0 {
 431                         continue
 432                 }
 433                 opts := calls[0].Options.(arvados.ListOptions)
 434                 c.Check(opts.Limit, check.Equals, trial.limit)
 435         }
 436 }