From: Tom Clegg Date: Fri, 13 May 2022 20:52:43 +0000 (-0400) Subject: 18947: Merge branch 'main' X-Git-Tag: 2.5.0~175^2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/f6a29cc714dd3409f865ceb799a886dd0b5d8152?hp=-c 18947: Merge branch 'main' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- f6a29cc714dd3409f865ceb799a886dd0b5d8152 diff --combined build/run-build-packages.sh index c4415b4697,3771662423..3e1ed6a94d --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@@ -248,13 -248,13 +248,13 @@@ package_go_binary cmd/arvados-server ar "Provide authenticated http access to Arvados-hosted git repositories" package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \ "Dispatch Crunch containers on the local system" -package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \ +package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \ "Dispatch Crunch containers to a SLURM cluster" package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \ "Supervise a single Crunch container" package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \ "Gather cpu/memory/network statistics of running Crunch jobs" - package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \ + package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \ "Check health of all Arvados cluster services" package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" diff --combined doc/admin/upgrading.html.textile.liquid index 25cb26b40d,efce633e9e..3f6009a803 --- a/doc/admin/upgrading.html.textile.liquid +++ b/doc/admin/upgrading.html.textile.liquid @@@ -32,14 -32,27 +32,31 @@@ h2(#main). development main (as of 2022 "previous: Upgrading to 2.4.0":#v2_4_0 +h3. Slurm dispatcher requires configuration update + +If you use the Slurm dispatcher (@crunch-dispatch-slurm@) you must add a @Services.DispatchSLURM.InternalURLs@ section to your configuration file, as shown on the "updated install page":{{site.baseurl}}/install/crunch2-slurm/install-dispatch.html. + + h3. New proxy parameters for arvados-controller + + We now recommend disabling nginx proxy caching for arvados-controller, to avoid truncation of large responses. + + In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@). + + 
+     proxy_max_temp_file_size 0;
+     proxy_request_buffering  off;
+     proxy_buffering          off;
+     proxy_http_version       1.1;
+
+ h3. Now recommending Singularity 3.9.9 The compute image "build script":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html now installs Singularity 3.9.9 instead of 3.7.4. The newer version includes a bugfix that should resolve "intermittent loopback device errors":https://dev.arvados.org/issues/18489 when running containers. + h3. Changes to @arvados-cwl-runner --create-workflow@ and @--update-workflow@ + + When using @arvados-cwl-runner --create-workflow@ or @--update-workflow@, by default it will now make a copy of all collection and Docker image dependencies in the target project. Running workflows retains the old behavior (use the dependencies wherever they are found). The can be controlled explicit with @--copy-deps@ and @--no-copy-deps@. + h2(#v2_4_0). v2.4.0 (2022-04-08) "previous: Upgrading to 2.3.1":#v2_3_1 diff --combined sdk/go/arvados/config.go index ace33c9ff0,6a90c30ce4..319fa1a38f --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@@ -348,7 -348,6 +348,7 @@@ type Services struct Controller Service DispatchCloud Service DispatchLSF Service + DispatchSLURM Service GitHTTP Service GitSSH Service Health Service @@@ -425,11 -424,11 +425,11 @@@ type CUDAFeatures struct } type InstanceType struct { - Name string + Name string `json:"-"` ProviderType string VCPUs int RAM ByteSize - Scratch ByteSize + Scratch ByteSize `json:"-"` IncludedScratch ByteSize AddedScratch ByteSize Price float64 @@@ -529,49 -528,23 +529,23 @@@ type InstanceTypeMap map[string]Instanc var errDuplicateInstanceTypeName = errors.New("duplicate instance type name") - // UnmarshalJSON handles old config files that provide an array of - // instance types instead of a hash. + // UnmarshalJSON does special handling of InstanceTypes: + // * populate computed fields (Name and Scratch) + // * error out if InstancesTypes are populated as an array, which was + // deprecated in Arvados 1.2.0 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error { fixup := func(t InstanceType) (InstanceType, error) { if t.ProviderType == "" { t.ProviderType = t.Name } - if t.Scratch == 0 { - t.Scratch = t.IncludedScratch + t.AddedScratch - } else if t.AddedScratch == 0 { - t.AddedScratch = t.Scratch - t.IncludedScratch - } else if t.IncludedScratch == 0 { - t.IncludedScratch = t.Scratch - t.AddedScratch - } - - if t.Scratch != (t.IncludedScratch + t.AddedScratch) { - return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name) - } + // If t.Scratch is set in the configuration file, it will be ignored and overwritten. + // It will also generate a "deprecated or unknown config entry" warning. + t.Scratch = t.IncludedScratch + t.AddedScratch return t, nil } if len(data) > 0 && data[0] == '[' { - var arr []InstanceType - err := json.Unmarshal(data, &arr) - if err != nil { - return err - } - if len(arr) == 0 { - *it = nil - return nil - } - *it = make(map[string]InstanceType, len(arr)) - for _, t := range arr { - if _, ok := (*it)[t.Name]; ok { - return errDuplicateInstanceTypeName - } - t, err := fixup(t) - if err != nil { - return err - } - (*it)[t.Name] = t - } - return nil + return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html") } var hash map[string]InstanceType err := json.Unmarshal(data, &hash) @@@ -632,7 -605,6 +606,7 @@@ const ServiceNameController ServiceName = "arvados-controller" ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud" ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf" + ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm" ServiceNameGitHTTP ServiceName = "arvados-git-httpd" ServiceNameHealth ServiceName = "arvados-health" ServiceNameKeepbalance ServiceName = "keep-balance" @@@ -652,7 -624,6 +626,7 @@@ func (svcs Services) Map() map[ServiceN ServiceNameController: svcs.Controller, ServiceNameDispatchCloud: svcs.DispatchCloud, ServiceNameDispatchLSF: svcs.DispatchLSF, + ServiceNameDispatchSLURM: svcs.DispatchSLURM, ServiceNameGitHTTP: svcs.GitHTTP, ServiceNameHealth: svcs.Health, ServiceNameKeepbalance: svcs.Keepbalance, diff --combined sdk/go/health/aggregator_test.go index 414902089e,5f60cf67f3..481054c4de --- a/sdk/go/health/aggregator_test.go +++ b/sdk/go/health/aggregator_test.go @@@ -220,6 -220,40 +220,40 @@@ func (s *AggregatorSuite) TestConfigMis s.checkOK(c) } + func (s *AggregatorSuite) TestClockSkew(c *check.C) { + // srv1: report real wall clock time + handler1 := healthyHandler{} + srv1, listen1 := s.stubServer(&handler1) + defer srv1.Close() + // srv2: report near-future time + handler2 := healthyHandler{headerDate: time.Now().Add(3 * time.Second)} + srv2, listen2 := s.stubServer(&handler2) + defer srv2.Close() + // srv3: report far-future time + handler3 := healthyHandler{headerDate: time.Now().Add(3*time.Minute + 3*time.Second)} + srv3, listen3 := s.stubServer(&handler3) + defer srv3.Close() + + s.setAllServiceURLs(listen1) + + // near-future time => OK + s.resp = httptest.NewRecorder() + arvadostest.SetServiceURL(&s.handler.Cluster.Services.DispatchCloud, + "http://localhost"+listen2+"/") + s.handler.ServeHTTP(s.resp, s.req) + s.checkOK(c) + + // far-future time => error + s.resp = httptest.NewRecorder() + arvadostest.SetServiceURL(&s.handler.Cluster.Services.WebDAV, + "http://localhost"+listen3+"/") + s.handler.ServeHTTP(s.resp, s.req) + resp := s.checkUnhealthy(c) + if c.Check(len(resp.Errors) > 0, check.Equals, true) { + c.Check(resp.Errors[0], check.Matches, `clock skew detected: maximum timestamp spread is 3m.* \(exceeds warning threshold of 1m\)`) + } + } + func (s *AggregatorSuite) TestPingTimeout(c *check.C) { s.handler.timeout = arvados.Duration(100 * time.Millisecond) srv, listen := s.stubServer(&slowHandler{}) @@@ -293,7 -327,6 +327,7 @@@ func (s *AggregatorSuite) setAllService &svcs.Controller, &svcs.DispatchCloud, &svcs.DispatchLSF, + &svcs.DispatchSLURM, &svcs.GitHTTP, &svcs.Keepbalance, &svcs.Keepproxy, @@@ -322,9 -355,13 +356,13 @@@ func (*unhealthyHandler) ServeHTTP(res type healthyHandler struct { configHash string configTime time.Time + headerDate time.Time } func (h *healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if !h.headerDate.IsZero() { + resp.Header().Set("Date", h.headerDate.Format(time.RFC1123)) + } authOK := req.Header.Get("Authorization") == "Bearer "+arvadostest.ManagementToken if req.URL.Path == "/_health/ping" { if !authOK {