1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
14 "git.curoverse.com/arvados.git/sdk/go/arvados"
15 "git.curoverse.com/arvados.git/sdk/go/httpserver"
18 //go:generate go run generate.go
20 // CollectionList is used as a template to auto-generate List()
21 // methods for other types; see generate.go.
23 func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
25 var merged arvados.CollectionList
26 err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
27 cl, err := backend.CollectionList(ctx, options)
33 if len(merged.Items) == 0 {
36 merged.Items = append(merged.Items, cl.Items...)
38 uuids := make([]string, 0, len(cl.Items))
39 for _, item := range cl.Items {
40 uuids = append(uuids, item.UUID)
44 sort.Slice(merged.Items, func(i, j int) bool { return merged.Items[i].UUID < merged.Items[j].UUID })
45 if merged.Items == nil {
46 // Return empty results as [], not null
47 // (https://github.com/golang/go/issues/27589 might be
48 // a better solution in the future)
49 merged.Items = []arvados.Collection{}
54 // Call fn on one or more local/remote backends if opts indicates a
55 // federation-wide list query, i.e.:
57 // * There is at least one filter of the form
58 // ["uuid","in",[a,b,c,...]] or ["uuid","=",a]
60 // * One or more of the supplied UUIDs (a,b,c,...) has a non-local
63 // * There are no other filters
65 // (If opts doesn't indicate a federation-wide list query, fn is just
66 // called once with the local backend.)
68 // fn is called more than once only if the query meets the following
77 // * Each filter must be either "uuid = ..." or "uuid in [...]".
79 // * The maximum possible response size (total number of objects that
80 // could potentially be matched by all of the specified filters)
81 // exceeds the local cluster's response page size limit.
83 // If the query involves multiple backends but doesn't meet these
84 // restrictions, an error is returned without calling fn.
86 // Thus, the caller can assume that either:
88 // * splitListRequest() returns an error, or
90 // * fn is called exactly once, or
92 // * fn is called more than once, with options that satisfy the above
95 // Each call to fn indicates a single (local or remote) backend and a
96 // corresponding options argument suitable for sending to that
98 func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions, fn func(context.Context, string, arvados.API, arvados.ListOptions) ([]string, error)) error {
100 var matchAllFilters map[string]bool
101 for _, f := range opts.Filters {
102 matchThisFilter := map[string]bool{}
103 if f.Attr != "uuid" {
107 if f.Operator == "=" {
108 if uuid, ok := f.Operand.(string); ok {
109 matchThisFilter[uuid] = true
111 return httpErrorf(http.StatusBadRequest, "invalid operand type %T for filter %q", f.Operand, f)
113 } else if f.Operator == "in" {
114 if operand, ok := f.Operand.([]interface{}); ok {
115 // skip any elements that aren't
116 // strings (thus can't match a UUID,
117 // thus can't affect the response).
118 for _, v := range operand {
119 if uuid, ok := v.(string); ok {
120 matchThisFilter[uuid] = true
123 } else if strings, ok := f.Operand.([]string); ok {
124 for _, uuid := range strings {
125 matchThisFilter[uuid] = true
128 return httpErrorf(http.StatusBadRequest, "invalid operand type %T in filter %q", f.Operand, f)
135 if matchAllFilters == nil {
136 matchAllFilters = matchThisFilter
138 // Reduce matchAllFilters to the intersection
139 // of matchAllFilters ∩ matchThisFilter.
140 for uuid := range matchAllFilters {
141 if !matchThisFilter[uuid] {
142 delete(matchAllFilters, uuid)
148 if matchAllFilters == nil {
149 // Not filtering by UUID at all; just query the local
151 _, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
155 // Collate UUIDs in matchAllFilters by remote cluster ID --
156 // e.g., todoByRemote["aaaaa"]["aaaaa-4zz18-000000000000000"]
157 // will be true -- and count the total number of UUIDs we're
158 // filtering on, so we can compare it to our max page size
161 todoByRemote := map[string]map[string]bool{}
162 for uuid := range matchAllFilters {
164 // Cannot match anything, just drop it
166 if todoByRemote[uuid[:5]] == nil {
167 todoByRemote[uuid[:5]] = map[string]bool{}
169 todoByRemote[uuid[:5]][uuid] = true
174 if len(todoByRemote) > 1 {
176 return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
178 if opts.Count != "none" {
179 return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
181 if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
182 return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
184 if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
185 return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
187 selectingUUID := false
188 for _, attr := range opts.Select {
193 if opts.Select != nil && !selectingUUID {
194 return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with a select parameter that does not include uuid")
198 ctx, cancel := context.WithCancel(ctx)
200 errs := make(chan error, len(todoByRemote))
201 for clusterID, todo := range todoByRemote {
202 go func(clusterID string, todo map[string]bool) {
203 // This goroutine sends exactly one value to
205 batch := make([]string, 0, len(todo))
206 for uuid := range todo {
207 batch = append(batch, uuid)
210 var backend arvados.API
211 if clusterID == conn.cluster.ClusterID {
213 } else if backend = conn.remotes[clusterID]; backend == nil {
214 errs <- httpErrorf(http.StatusNotFound, "cannot execute federated list query: no proxy available for cluster %q", clusterID)
219 if len(batch) > len(todo) {
220 // Reduce batch to just the todo's
222 for uuid := range todo {
223 batch = append(batch, uuid)
226 remoteOpts.Filters = []arvados.Filter{{"uuid", "in", batch}}
228 done, err := fn(ctx, clusterID, backend, remoteOpts)
234 for _, uuid := range done {
235 if _, ok := todo[uuid]; ok {
241 errs <- httpErrorf(http.StatusBadGateway, "cannot make progress in federated list query: cluster %q returned none of the requested UUIDs", clusterID)
249 // Wait for all goroutines to return, then return the first
250 // non-nil error, if any.
252 for range todoByRemote {
253 if err := <-errs; err != nil && firstErr == nil {
255 // Signal to any remaining fn() calls that
256 // further effort is futile.
263 func httpErrorf(code int, format string, args ...interface{}) error {
264 return httpserver.ErrorWithStatus(fmt.Errorf(format, args...), code)