lib/controller/federation/list_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package federation
   6
   7 import (
   8         "context"
   9         "fmt"
  10         "net/http"
  11         "net/url"
  12         "os"
  13         "testing"
  14
  15         "git.curoverse.com/arvados.git/lib/controller/router"
  16         "git.curoverse.com/arvados.git/lib/controller/rpc"
  17         "git.curoverse.com/arvados.git/sdk/go/arvados"
  18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  19         "git.curoverse.com/arvados.git/sdk/go/auth"
  20         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  21         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 // Gocheck boilerplate
  26 func Test(t *testing.T) {
  27         check.TestingT(t)
  28 }
  29
  30 var (
  31         _ = check.Suite(&FederationSuite{})
  32         _ = check.Suite(&CollectionListSuite{})
  33 )
  34
  35 type FederationSuite struct {
  36         cluster *arvados.Cluster
  37         ctx     context.Context
  38         fed     *Conn
  39 }
  40
  41 func (s *FederationSuite) SetUpTest(c *check.C) {
  42         s.cluster = &arvados.Cluster{
  43                 ClusterID: "aaaaa",
  44                 RemoteClusters: map[string]arvados.RemoteCluster{
  45                         "aaaaa": arvados.RemoteCluster{
  46                                 Host: os.Getenv("ARVADOS_API_HOST"),
  47                         },
  48                 },
  49         }
  50         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  51         s.cluster.TLS.Insecure = true
  52         s.cluster.API.MaxItemsPerResponse = 3
  53
  54         ctx := context.Background()
  55         ctx = ctxlog.Context(ctx, ctxlog.TestLogger(c))
  56         ctx = auth.NewContext(ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  57         s.ctx = ctx
  58
  59         s.fed = New(s.cluster)
  60 }
  61
  62 func (s *FederationSuite) addDirectRemote(c *check.C, id string, backend arvados.API) {
  63         s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
  64                 Host: "in-process.local",
  65         }
  66         s.fed.remotes[id] = backend
  67 }
  68
  69 func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend arvados.API) {
  70         srv := httpserver.Server{Addr: ":"}
  71         srv.Handler = router.New(backend)
  72         c.Check(srv.Start(), check.IsNil)
  73         s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
  74                 Host:  srv.Addr,
  75                 Proxy: true,
  76         }
  77         s.fed.remotes[id] = rpc.NewConn(id, &url.URL{Scheme: "http", Host: srv.Addr}, true, saltedTokenProvider(s.fed.local, id))
  78 }
  79
  80 type collectionLister struct {
  81         arvadostest.APIStub
  82         ItemsToReturn []arvados.Collection
  83         MaxPageSize   int
  84 }
  85
  86 func (cl *collectionLister) matchFilters(c arvados.Collection, filters []arvados.Filter) bool {
  87 nextfilter:
  88         for _, f := range filters {
  89                 if f.Attr == "uuid" && f.Operator == "=" {
  90                         s, ok := f.Operand.(string)
  91                         if ok && s == c.UUID {
  92                                 continue nextfilter
  93                         }
  94                 } else if f.Attr == "uuid" && f.Operator == "in" {
  95                         if operand, ok := f.Operand.([]string); ok {
  96                                 for _, s := range operand {
  97                                         if s == c.UUID {
  98                                                 continue nextfilter
  99                                         }
 100                                 }
 101                         } else if operand, ok := f.Operand.([]interface{}); ok {
 102                                 for _, s := range operand {
 103                                         if s, ok := s.(string); ok && s == c.UUID {
 104                                                 continue nextfilter
 105                                         }
 106                                 }
 107                         }
 108                 }
 109                 return false
 110         }
 111         return true
 112 }
 113
 114 func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.ListOptions) (resp arvados.CollectionList, _ error) {
 115         cl.APIStub.CollectionList(ctx, options)
 116         for _, c := range cl.ItemsToReturn {
 117                 if cl.MaxPageSize > 0 && len(resp.Items) >= cl.MaxPageSize {
 118                         break
 119                 }
 120                 if cl.matchFilters(c, options.Filters) {
 121                         resp.Items = append(resp.Items, c)
 122                 }
 123         }
 124         return
 125 }
 126
 127 type CollectionListSuite struct {
 128         FederationSuite
 129         ids      []string   // aaaaa, bbbbb, ccccc
 130         uuids    [][]string // [[aa-*, aa-*, aa-*], [bb-*, bb-*, ...], ...]
 131         backends []*collectionLister
 132 }
 133
 134 func (s *CollectionListSuite) SetUpTest(c *check.C) {
 135         s.FederationSuite.SetUpTest(c)
 136
 137         s.ids = nil
 138         s.uuids = nil
 139         s.backends = nil
 140         for i, id := range []string{"aaaaa", "bbbbb", "ccccc"} {
 141                 cl := &collectionLister{}
 142                 s.ids = append(s.ids, id)
 143                 s.uuids = append(s.uuids, nil)
 144                 for j := 0; j < 5; j++ {
 145                         uuid := fmt.Sprintf("%s-4zz18-%s%010d", id, id, j)
 146                         s.uuids[i] = append(s.uuids[i], uuid)
 147                         cl.ItemsToReturn = append(cl.ItemsToReturn, arvados.Collection{
 148                                 UUID: uuid,
 149                         })
 150                 }
 151                 s.backends = append(s.backends, cl)
 152                 if i == 0 {
 153                         s.fed.local = cl
 154                 } else if i%1 == 0 {
 155                         // call some backends directly via API
 156                         s.addDirectRemote(c, id, cl)
 157                 } else {
 158                         // call some backends through rpc->router->API
 159                         // to ensure nothing is lost in translation
 160                         s.addHTTPRemote(c, id, cl)
 161                 }
 162         }
 163 }
 164
 165 type listTrial struct {
 166         count        string
 167         limit        int
 168         offset       int
 169         order        []string
 170         filters      []arvados.Filter
 171         expectUUIDs  []string
 172         expectCalls  []int // number of API calls to backends
 173         expectStatus int
 174 }
 175
 176 func (s *CollectionListSuite) TestCollectionListOneLocal(c *check.C) {
 177         s.test(c, listTrial{
 178                 count:       "none",
 179                 limit:       -1,
 180                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
 181                 expectUUIDs: []string{s.uuids[0][0]},
 182                 expectCalls: []int{1, 0, 0},
 183         })
 184 }
 185
 186 func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
 187         s.test(c, listTrial{
 188                 count:       "none",
 189                 limit:       -1,
 190                 filters:     []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
 191                 expectUUIDs: []string{s.uuids[1][0]},
 192                 expectCalls: []int{0, 1, 0},
 193         })
 194 }
 195
 196 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
 197         s.test(c, listTrial{
 198                 count:       "none",
 199                 limit:       -1,
 200                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0]}}},
 201                 expectUUIDs: []string{s.uuids[0][0]},
 202                 expectCalls: []int{1, 0, 0},
 203         })
 204 }
 205
 206 func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *check.C) {
 207         s.test(c, listTrial{
 208                 count:       "none",
 209                 limit:       -1,
 210                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[1][1]}}},
 211                 expectUUIDs: []string{s.uuids[1][1]},
 212                 expectCalls: []int{0, 1, 0},
 213         })
 214 }
 215
 216 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
 217         s.test(c, listTrial{
 218                 count:       "none",
 219                 limit:       -1,
 220                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
 221                 expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]},
 222                 expectCalls: []int{1, 1, 0},
 223         })
 224 }
 225
 226 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
 227         s.test(c, listTrial{
 228                 count:       "none",
 229                 limit:       -1,
 230                 filters:     []arvados.Filter{{"uuid", "in", []string{s.uuids[2][0], s.uuids[1][0]}}},
 231                 expectUUIDs: []string{s.uuids[1][0], s.uuids[2][0]},
 232                 expectCalls: []int{0, 1, 1},
 233         })
 234 }
 235
 236 func (s *CollectionListSuite) TestCollectionListSatisfyAllFilters(c *check.C) {
 237         s.cluster.API.MaxItemsPerResponse = 2
 238         s.test(c, listTrial{
 239                 count: "none",
 240                 limit: -1,
 241                 filters: []arvados.Filter{
 242                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][1], s.uuids[2][0], s.uuids[2][1], s.uuids[2][2]}},
 243                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][2], s.uuids[2][1]}},
 244                 },
 245                 expectUUIDs: []string{s.uuids[0][0], s.uuids[2][1]},
 246                 expectCalls: []int{1, 0, 1},
 247         })
 248 }
 249
 250 func (s *CollectionListSuite) TestCollectionListEmptySet(c *check.C) {
 251         s.test(c, listTrial{
 252                 count:       "none",
 253                 limit:       -1,
 254                 filters:     []arvados.Filter{{"uuid", "in", []string{}}},
 255                 expectUUIDs: []string{},
 256                 expectCalls: []int{0, 0, 0},
 257         })
 258 }
 259
 260 func (s *CollectionListSuite) TestCollectionListUnmatchableUUID(c *check.C) {
 261         s.test(c, listTrial{
 262                 count: "none",
 263                 limit: -1,
 264                 filters: []arvados.Filter{
 265                         {"uuid", "in", []string{s.uuids[0][0], "abcdefg"}},
 266                         {"uuid", "in", []string{s.uuids[0][0], "bbbbb-4zz18-bogus"}},
 267                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-bogus"}},
 268                 },
 269                 expectUUIDs: []string{s.uuids[0][0]},
 270                 expectCalls: []int{1, 0, 0},
 271         })
 272 }
 273
 274 func (s *CollectionListSuite) TestCollectionListMultiPage(c *check.C) {
 275         for i := range s.backends {
 276                 s.uuids[i] = s.uuids[i][:3]
 277                 s.backends[i].ItemsToReturn = s.backends[i].ItemsToReturn[:3]
 278         }
 279         s.cluster.API.MaxItemsPerResponse = 9
 280         for _, stub := range s.backends {
 281                 stub.MaxPageSize = 2
 282         }
 283         allUUIDs := append(append(append([]string(nil), s.uuids[0]...), s.uuids[1]...), s.uuids[2]...)
 284         s.test(c, listTrial{
 285                 count:       "none",
 286                 limit:       -1,
 287                 filters:     []arvados.Filter{{"uuid", "in", append([]string(nil), allUUIDs...)}},
 288                 expectUUIDs: allUUIDs,
 289                 expectCalls: []int{2, 2, 2},
 290         })
 291 }
 292
 293 func (s *CollectionListSuite) TestCollectionListMultiSiteExtraFilters(c *check.C) {
 294         // not [yet] supported
 295         s.test(c, listTrial{
 296                 count: "none",
 297                 limit: -1,
 298                 filters: []arvados.Filter{
 299                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 300                         {"uuid", "is_a", "teapot"},
 301                 },
 302                 expectCalls:  []int{0, 0, 0},
 303                 expectStatus: http.StatusBadRequest,
 304         })
 305 }
 306
 307 func (s *CollectionListSuite) TestCollectionListMultiSiteWithCount(c *check.C) {
 308         for _, count := range []string{"", "exact"} {
 309                 s.test(c, listTrial{
 310                         count: count,
 311                         limit: -1,
 312                         filters: []arvados.Filter{
 313                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 314                                 {"uuid", "is_a", "teapot"},
 315                         },
 316                         expectCalls:  []int{0, 0, 0},
 317                         expectStatus: http.StatusBadRequest,
 318                 })
 319         }
 320 }
 321
 322 func (s *CollectionListSuite) TestCollectionListMultiSiteWithLimit(c *check.C) {
 323         for _, limit := range []int{0, 1, 2} {
 324                 s.test(c, listTrial{
 325                         count: "none",
 326                         limit: limit,
 327                         filters: []arvados.Filter{
 328                                 {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 329                                 {"uuid", "is_a", "teapot"},
 330                         },
 331                         expectCalls:  []int{0, 0, 0},
 332                         expectStatus: http.StatusBadRequest,
 333                 })
 334         }
 335 }
 336
 337 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOffset(c *check.C) {
 338         s.test(c, listTrial{
 339                 count:  "none",
 340                 limit:  -1,
 341                 offset: 1,
 342                 filters: []arvados.Filter{
 343                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 344                         {"uuid", "is_a", "teapot"},
 345                 },
 346                 expectCalls:  []int{0, 0, 0},
 347                 expectStatus: http.StatusBadRequest,
 348         })
 349 }
 350
 351 func (s *CollectionListSuite) TestCollectionListMultiSiteWithOrder(c *check.C) {
 352         s.test(c, listTrial{
 353                 count: "none",
 354                 limit: -1,
 355                 order: []string{"uuid desc"},
 356                 filters: []arvados.Filter{
 357                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 358                         {"uuid", "is_a", "teapot"},
 359                 },
 360                 expectCalls:  []int{0, 0, 0},
 361                 expectStatus: http.StatusBadRequest,
 362         })
 363 }
 364
 365 func (s *CollectionListSuite) TestCollectionListInvalidFilters(c *check.C) {
 366         s.test(c, listTrial{
 367                 count: "none",
 368                 limit: -1,
 369                 filters: []arvados.Filter{
 370                         {"uuid", "in", "teapot"},
 371                 },
 372                 expectCalls:  []int{0, 0, 0},
 373                 expectStatus: http.StatusBadRequest,
 374         })
 375 }
 376
 377 func (s *CollectionListSuite) TestCollectionListRemoteUnknown(c *check.C) {
 378         s.test(c, listTrial{
 379                 count: "none",
 380                 limit: -1,
 381                 filters: []arvados.Filter{
 382                         {"uuid", "in", []string{s.uuids[0][0], "bogus-4zz18-000001111122222"}},
 383                 },
 384                 expectStatus: http.StatusNotFound,
 385         })
 386 }
 387
 388 func (s *CollectionListSuite) TestCollectionListRemoteError(c *check.C) {
 389         s.addDirectRemote(c, "bbbbb", &arvadostest.APIStub{})
 390         s.test(c, listTrial{
 391                 count: "none",
 392                 limit: -1,
 393                 filters: []arvados.Filter{
 394                         {"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}},
 395                 },
 396                 expectStatus: http.StatusBadGateway,
 397         })
 398 }
 399
 400 func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
 401         resp, err := s.fed.CollectionList(s.ctx, arvados.ListOptions{
 402                 Count:   trial.count,
 403                 Limit:   trial.limit,
 404                 Offset:  trial.offset,
 405                 Order:   trial.order,
 406                 Filters: trial.filters,
 407         })
 408         if trial.expectStatus != 0 {
 409                 c.Assert(err, check.NotNil)
 410                 err, _ := err.(interface{ HTTPStatus() int })
 411                 c.Assert(err, check.NotNil) // err must implement HTTPStatus()
 412                 c.Check(err.HTTPStatus(), check.Equals, trial.expectStatus)
 413                 c.Logf("returned error is %#v", err)
 414                 c.Logf("returned error string is %q", err)
 415         } else {
 416                 c.Check(err, check.IsNil)
 417                 var expectItems []arvados.Collection
 418                 for _, uuid := range trial.expectUUIDs {
 419                         expectItems = append(expectItems, arvados.Collection{UUID: uuid})
 420                 }
 421                 c.Check(resp, check.DeepEquals, arvados.CollectionList{
 422                         Items: expectItems,
 423                 })
 424         }
 425
 426         for i, stub := range s.backends {
 427                 if i >= len(trial.expectCalls) {
 428                         break
 429                 }
 430                 calls := stub.Calls(nil)
 431                 c.Check(calls, check.HasLen, trial.expectCalls[i])
 432                 if len(calls) == 0 {
 433                         continue
 434                 }
 435                 opts := calls[0].Options.(arvados.ListOptions)
 436                 c.Check(opts.Limit, check.Equals, -1)
 437         }
 438 }