"Provide authenticated http access to Arvados-hosted git repositories"
package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
"Dispatch Crunch containers on the local system"
-package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \
"Dispatch Crunch containers to a SLURM cluster"
package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
"Supervise a single Crunch container"
package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \
"Gather cpu/memory/network statistics of running Crunch jobs"
- package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \
+ package_go_binary cmd/arvados-server arvados-health "$FORMAT" "$ARCH" \
"Check health of all Arvados cluster services"
package_go_binary cmd/arvados-server keep-balance "$FORMAT" "$ARCH" \
"Rebalance and garbage-collect data blocks stored in Arvados Keep"
"previous: Upgrading to 2.4.0":#v2_4_0
+h3. Slurm dispatcher requires configuration update
+
+If you use the Slurm dispatcher (@crunch-dispatch-slurm@) you must add a @Services.DispatchSLURM.InternalURLs@ section to your configuration file, as shown on the "updated install page":{{site.baseurl}}/install/crunch2-slurm/install-dispatch.html.
+
+ h3. New proxy parameters for arvados-controller
+
+ We now recommend disabling nginx proxy caching for arvados-controller, to avoid truncation of large responses.
+
+ In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@).
+
+ <pre>
+ proxy_max_temp_file_size 0;
+ proxy_request_buffering off;
+ proxy_buffering off;
+ proxy_http_version 1.1;
+ </pre>
+
h3. Now recommending Singularity 3.9.9
The compute image "build script":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html now installs Singularity 3.9.9 instead of 3.7.4. The newer version includes a bugfix that should resolve "intermittent loopback device errors":https://dev.arvados.org/issues/18489 when running containers.
+ h3. Changes to @arvados-cwl-runner --create-workflow@ and @--update-workflow@
+
+ When using @arvados-cwl-runner --create-workflow@ or @--update-workflow@, by default it will now make a copy of all collection and Docker image dependencies in the target project. Running workflows retains the old behavior (use the dependencies wherever they are found). The can be controlled explicit with @--copy-deps@ and @--no-copy-deps@.
+
h2(#v2_4_0). v2.4.0 (2022-04-08)
"previous: Upgrading to 2.3.1":#v2_3_1
Controller Service
DispatchCloud Service
DispatchLSF Service
+ DispatchSLURM Service
GitHTTP Service
GitSSH Service
Health Service
}
type InstanceType struct {
- Name string
+ Name string `json:"-"`
ProviderType string
VCPUs int
RAM ByteSize
- Scratch ByteSize
+ Scratch ByteSize `json:"-"`
IncludedScratch ByteSize
AddedScratch ByteSize
Price float64
var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
- // UnmarshalJSON handles old config files that provide an array of
- // instance types instead of a hash.
+ // UnmarshalJSON does special handling of InstanceTypes:
+ // * populate computed fields (Name and Scratch)
+ // * error out if InstancesTypes are populated as an array, which was
+ // deprecated in Arvados 1.2.0
func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
fixup := func(t InstanceType) (InstanceType, error) {
if t.ProviderType == "" {
t.ProviderType = t.Name
}
- if t.Scratch == 0 {
- t.Scratch = t.IncludedScratch + t.AddedScratch
- } else if t.AddedScratch == 0 {
- t.AddedScratch = t.Scratch - t.IncludedScratch
- } else if t.IncludedScratch == 0 {
- t.IncludedScratch = t.Scratch - t.AddedScratch
- }
-
- if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
- return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
- }
+ // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
+ // It will also generate a "deprecated or unknown config entry" warning.
+ t.Scratch = t.IncludedScratch + t.AddedScratch
return t, nil
}
if len(data) > 0 && data[0] == '[' {
- var arr []InstanceType
- err := json.Unmarshal(data, &arr)
- if err != nil {
- return err
- }
- if len(arr) == 0 {
- *it = nil
- return nil
- }
- *it = make(map[string]InstanceType, len(arr))
- for _, t := range arr {
- if _, ok := (*it)[t.Name]; ok {
- return errDuplicateInstanceTypeName
- }
- t, err := fixup(t)
- if err != nil {
- return err
- }
- (*it)[t.Name] = t
- }
- return nil
+ return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
}
var hash map[string]InstanceType
err := json.Unmarshal(data, &hash)
ServiceNameController ServiceName = "arvados-controller"
ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
+ ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
ServiceNameHealth ServiceName = "arvados-health"
ServiceNameKeepbalance ServiceName = "keep-balance"
ServiceNameController: svcs.Controller,
ServiceNameDispatchCloud: svcs.DispatchCloud,
ServiceNameDispatchLSF: svcs.DispatchLSF,
+ ServiceNameDispatchSLURM: svcs.DispatchSLURM,
ServiceNameGitHTTP: svcs.GitHTTP,
ServiceNameHealth: svcs.Health,
ServiceNameKeepbalance: svcs.Keepbalance,
s.checkOK(c)
}
+ func (s *AggregatorSuite) TestClockSkew(c *check.C) {
+ // srv1: report real wall clock time
+ handler1 := healthyHandler{}
+ srv1, listen1 := s.stubServer(&handler1)
+ defer srv1.Close()
+ // srv2: report near-future time
+ handler2 := healthyHandler{headerDate: time.Now().Add(3 * time.Second)}
+ srv2, listen2 := s.stubServer(&handler2)
+ defer srv2.Close()
+ // srv3: report far-future time
+ handler3 := healthyHandler{headerDate: time.Now().Add(3*time.Minute + 3*time.Second)}
+ srv3, listen3 := s.stubServer(&handler3)
+ defer srv3.Close()
+
+ s.setAllServiceURLs(listen1)
+
+ // near-future time => OK
+ s.resp = httptest.NewRecorder()
+ arvadostest.SetServiceURL(&s.handler.Cluster.Services.DispatchCloud,
+ "http://localhost"+listen2+"/")
+ s.handler.ServeHTTP(s.resp, s.req)
+ s.checkOK(c)
+
+ // far-future time => error
+ s.resp = httptest.NewRecorder()
+ arvadostest.SetServiceURL(&s.handler.Cluster.Services.WebDAV,
+ "http://localhost"+listen3+"/")
+ s.handler.ServeHTTP(s.resp, s.req)
+ resp := s.checkUnhealthy(c)
+ if c.Check(len(resp.Errors) > 0, check.Equals, true) {
+ c.Check(resp.Errors[0], check.Matches, `clock skew detected: maximum timestamp spread is 3m.* \(exceeds warning threshold of 1m\)`)
+ }
+ }
+
func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
s.handler.timeout = arvados.Duration(100 * time.Millisecond)
srv, listen := s.stubServer(&slowHandler{})
&svcs.Controller,
&svcs.DispatchCloud,
&svcs.DispatchLSF,
+ &svcs.DispatchSLURM,
&svcs.GitHTTP,
&svcs.Keepbalance,
&svcs.Keepproxy,
type healthyHandler struct {
configHash string
configTime time.Time
+ headerDate time.Time
}
func (h *healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if !h.headerDate.IsZero() {
+ resp.Header().Set("Date", h.headerDate.Format(time.RFC1123))
+ }
authOK := req.Header.Get("Authorization") == "Bearer "+arvadostest.ManagementToken
if req.URL.Path == "/_health/ping" {
if !authOK {
projects / arvados.git / commitdiff