Merge branch '16111-ssh-help' closes #16111

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/cmd/arvados-server/arvados-controller.service b/cmd/arvados-server/arvados-controller.service
index e85707473feefead0e51a50faace4bd947f5069f..1a43d7cd3a72bb31a1e1ef2152e152c75fe2c6aa 100644 (file)
--- a/cmd/arvados-server/arvados-controller.service
+++ b/cmd/arvados-server/arvados-controller.service
@@ -18,6 +18,8 @@ StartLimitIntervalSec=0
 Type=notify
 EnvironmentFile=-/etc/arvados/environment
 ExecStart=/usr/bin/arvados-controller
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/cmd/arvados-server/arvados-dispatch-cloud.service b/cmd/arvados-server/arvados-dispatch-cloud.service
index aa5cc3b4a5d033c3c163b09ad9a4ad411c3237b4..d3f476b7df725c2597f3c972e476d4a51ff86165 100644 (file)
--- a/cmd/arvados-server/arvados-dispatch-cloud.service
+++ b/cmd/arvados-server/arvados-dispatch-cloud.service
@@ -18,6 +18,8 @@ StartLimitIntervalSec=0
 Type=notify
 EnvironmentFile=-/etc/arvados/environment
 ExecStart=/usr/bin/arvados-dispatch-cloud
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index 175463c3253a36c73ff10bff6f62e3f97f261f3a..e08dbb283709cf1f202552a9ca39673e180f9da4 100644 (file)
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -141,7 +141,6 @@ To query multiple clusters, the list request must:
 
 * Have filters only matching @[["uuid", "in", [...]]@ or @["uuid", "=", "..."]@
 * Specify @count=none@
-* If @select@ is specified, it must include @uuid@
 * Not specify @limit@, @offset@ or @order@
 * Not request more items than the maximum response size
 

diff --git a/lib/controller/federation/list.go b/lib/controller/federation/list.go
index 97c201e742c9ee65eeae759c4e8d9bb63304c154..6ee813317417cd012d829387e10e04f2ea1dad17 100644 (file)
--- a/lib/controller/federation/list.go
+++ b/lib/controller/federation/list.go
@@ -84,7 +84,7 @@ func (conn *Conn) generated_CollectionList(ctx context.Context, options arvados.
 //
 // * len(Order)==0
 //
-// * Each filter must be either "uuid = ..." or "uuid in [...]".
+// * Each filter is either "uuid = ..." or "uuid in [...]".
 //
 // * The maximum possible response size (total number of objects that
 //   could potentially be matched by all of the specified filters)
@@ -181,28 +181,27 @@ func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions
                }
        }
 
-       if len(todoByRemote) > 1 {
-               if cannotSplit {
-                       return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
-               }
-               if opts.Count != "none" {
-                       return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
-               }
-               if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
-                       return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
-               }
-               if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
-                       return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
-               }
-               selectingUUID := false
-               for _, attr := range opts.Select {
-                       if attr == "uuid" {
-                               selectingUUID = true
-                       }
-               }
-               if opts.Select != nil && !selectingUUID {
-                       return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with a select parameter that does not include uuid")
-               }
+       if len(todoByRemote) == 0 {
+               return nil
+       }
+       if len(todoByRemote) == 1 && todoByRemote[conn.cluster.ClusterID] != nil {
+               // All UUIDs are local, so proxy a single request. The
+               // generic case has some limitations (see below) which
+               // we don't want to impose on local requests.
+               _, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
+               return err
+       }
+       if cannotSplit {
+               return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
+       }
+       if opts.Count != "none" {
+               return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
+       }
+       if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
+               return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
+       }
+       if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
+               return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
        }
 
        ctx, cancel := context.WithCancel(ctx)
@@ -225,6 +224,12 @@ func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions
                                return
                        }
                        remoteOpts := opts
+                       if remoteOpts.Select != nil {
+                               // We always need to select UUIDs to
+                               // use the response, even if our
+                               // caller doesn't.
+                               remoteOpts.Select = append([]string{"uuid"}, remoteOpts.Select...)
+                       }
                        for len(todo) > 0 {
                                if len(batch) > len(todo) {
                                        // Reduce batch to just the todo's

diff --git a/lib/controller/federation/list_test.go b/lib/controller/federation/list_test.go
index f61dd5476567588f84999f110cfa2c521ec96b54..ce84378a3cbb79d7a4ae894f966bc627bef2b08d 100644 (file)
--- a/lib/controller/federation/list_test.go
+++ b/lib/controller/federation/list_test.go
@@ -8,6 +8,7 @@ import (
        "context"
        "fmt"
        "net/http"
+       "reflect"
        "sort"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
@@ -61,6 +62,13 @@ func (cl *collectionLister) CollectionList(ctx context.Context, options arvados.
                        break
                }
                if cl.matchFilters(c, options.Filters) {
+                       if reflect.DeepEqual(options.Select, []string{"uuid", "name"}) {
+                               c = arvados.Collection{UUID: c.UUID, Name: c.Name}
+                       } else if reflect.DeepEqual(options.Select, []string{"name"}) {
+                               c = arvados.Collection{Name: c.Name}
+                       } else if len(options.Select) > 0 {
+                               panic(fmt.Sprintf("not implemented: options=%#v", options))
+                       }
                        resp.Items = append(resp.Items, c)
                }
        }
@@ -111,6 +119,7 @@ type listTrial struct {
        offset       int
        order        []string
        filters      []arvados.Filter
+       selectfields []string
        expectUUIDs  []string
        expectCalls  []int // number of API calls to backends
        expectStatus int
@@ -145,6 +154,17 @@ func (s *CollectionListSuite) TestCollectionListOneRemote(c *check.C) {
        })
 }
 
+func (s *CollectionListSuite) TestCollectionListOneLocalDeselectingUUID(c *check.C) {
+       s.test(c, listTrial{
+               count:        "none",
+               limit:        -1,
+               filters:      []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
+               selectfields: []string{"name"},
+               expectUUIDs:  []string{""}, // select=name is honored
+               expectCalls:  []int{1, 0, 0},
+       })
+}
+
 func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
        s.test(c, listTrial{
                count:       "none",
@@ -165,6 +185,17 @@ func (s *CollectionListSuite) TestCollectionListOneRemoteUsingInOperator(c *chec
        })
 }
 
+func (s *CollectionListSuite) TestCollectionListOneRemoteDeselectingUUID(c *check.C) {
+       s.test(c, listTrial{
+               count:        "none",
+               limit:        -1,
+               filters:      []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
+               selectfields: []string{"name"},
+               expectUUIDs:  []string{s.uuids[1][0]}, // uuid is returned, despite not being selected
+               expectCalls:  []int{0, 1, 0},
+       })
+}
+
 func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
        s.test(c, listTrial{
                count:       "none",
@@ -175,6 +206,17 @@ func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
        })
 }
 
+func (s *CollectionListSuite) TestCollectionListOneLocalOneRemoteDeselectingUUID(c *check.C) {
+       s.test(c, listTrial{
+               count:        "none",
+               limit:        -1,
+               filters:      []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
+               selectfields: []string{"name"},
+               expectUUIDs:  []string{s.uuids[0][0], s.uuids[1][0]}, // uuid is returned, despite not being selected
+               expectCalls:  []int{1, 1, 0},
+       })
+}
+
 func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
        s.test(c, listTrial{
                count:       "none",
@@ -356,6 +398,7 @@ func (s *CollectionListSuite) test(c *check.C, trial listTrial) {
                Offset:  trial.offset,
                Order:   trial.order,
                Filters: trial.filters,
+               Select:  trial.selectfields,
        })
        if trial.expectStatus != 0 {
                c.Assert(err, check.NotNil)

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
index 9ca1134ed8d5375d7c417af0b3ad684f34efa847..2af56c8d0c1e62b3c2a1d3eb5f0a5c1a65be7b4a 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
@@ -16,6 +16,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=notify
 ExecStart=/usr/bin/crunch-dispatch-slurm
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 LimitNOFILE=1000000

diff --git a/services/health/arvados-health.service b/services/health/arvados-health.service
index dac7c3a1949c9793cf0aaa2cbcf6b297d4a5a241..ca3744c28db265832229ac5a0635114396164b35 100644 (file)
--- a/services/health/arvados-health.service
+++ b/services/health/arvados-health.service
@@ -17,6 +17,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=simple
 ExecStart=/usr/bin/arvados-health
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 1b71fb4e44350bac913961e598494d0c01a333ab..0a38597e6caf28a3f1a4ba45a9b568c2920d56c8 100644 (file)
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -16,6 +16,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=simple
 ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=10s
 Nice=19

diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
index 23a2c659e0db7349af1a2c7221770523ebac6a2b..80978227770998fa2ac1b69d5149cfd4e2ca9530 100644 (file)
--- a/services/keep-web/keep-web.service
+++ b/services/keep-web/keep-web.service
@@ -16,6 +16,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=notify
 ExecStart=/usr/bin/keep-web
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/services/keepproxy/keepproxy.service b/services/keepproxy/keepproxy.service
index 1d0113e0e43a04a17fa2e7ae9657f1ed7bc9f1fa..4c63161a0d7a260ce7dd349f62214d6e7d162f36 100644 (file)
--- a/services/keepproxy/keepproxy.service
+++ b/services/keepproxy/keepproxy.service
@@ -16,6 +16,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=notify
 ExecStart=/usr/bin/keepproxy
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 2eba5efbfd242d46109b0f8dfa0092a59c618d62..7047f0e6b970a0c0a9598d6e680d5589425c08c1 100644 (file)
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -21,6 +21,8 @@ StartLimitIntervalSec=0
 Environment=GOGC=10
 Type=notify
 ExecStart=/usr/bin/keepstore
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1
 

diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
index 4d7874d42c26ac8ef03eb07ecc3df5063f195b05..36624c78779c02cfde829323551ca9c2cb19eda3 100644 (file)
--- a/services/ws/arvados-ws.service
+++ b/services/ws/arvados-ws.service
@@ -16,6 +16,8 @@ StartLimitIntervalSec=0
 [Service]
 Type=notify
 ExecStart=/usr/bin/arvados-ws
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
 Restart=always
 RestartSec=1