echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
- #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
else
echo "$test was't found on $file"
cleaned_test=$(echo $test | tr -d ",.:;/")
--- /dev/null
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% if site.current_version and site.current_version != 'main' %}
+{% assign branchname = site.current_version | slice: 0, 3 | append: '-dev' %}
+{% else %}
+{% assign branchname = 'main' %}
+{% endif %}
h2(#multi_host). Multi host install using the provision.sh script
-{% if site.current_version %}
-{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
-{% else %}
-{% assign branchname = 'main' %}
-{% endif %}
+{% include 'branchname' %}
This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
<b>NOTE: The single host installation is not recommended for production use.</b>
-{% if site.current_version %}
-{% assign branchname = site.current_version | slice: 1, 5 | append: '-dev' %}
-{% else %}
-{% assign branchname = 'main' %}
-{% endif %}
+{% include 'branchname' %}
This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git . The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
h2(#introduction). Introduction
+{% include 'branchname' %}
+
To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://git.arvados.org/arvados-formula.git which can help you get an Arvados cluster up and running.
Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a _master/minion_ setup (where a master node orchestrates and coordinates the configuration of nodes in an infrastructure) or <i>master-less</i>, where Saltstack is run locally in a node, with no communication with a master node.
"context"
"encoding/json"
"fmt"
+ "net"
"net/http"
"net/http/httptest"
"os"
s.testHandler = &Handler{Cluster: cluster}
s.testServer = newServerFromIntegrationTestEnv(c)
- s.testServer.Server.Handler = httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), s.log),
- httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler)))
+ s.testServer.Server.BaseContext = func(net.Listener) context.Context {
+ return ctxlog.Context(context.Background(), s.log)
+ }
+ s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler))
c.Assert(s.testServer.Start(), check.IsNil)
}
"fmt"
"io"
"io/ioutil"
+ "net"
"net/http"
"net/http/httptest"
"net/url"
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
s.testServer = newServerFromIntegrationTestEnv(c)
- s.testServer.Server.Handler = httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), s.log),
- httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler)))
+ s.testServer.Server.BaseContext = func(net.Listener) context.Context {
+ return ctxlog.Context(context.Background(), s.log)
+ }
+ s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler))
cluster.RemoteClusters = map[string]arvados.RemoteCluster{
"zzzzz": {
import (
"context"
+ "net"
"net/http"
"os"
"path/filepath"
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), log),
- httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
+ BaseContext: func(net.Listener) context.Context {
+ return ctxlog.Context(context.Background(), log)
+ },
+ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(handler)),
},
Addr: ":",
}
}
instrumented := httpserver.Instrument(reg, log,
- httpserver.HandlerWithContext(ctx,
+ httpserver.HandlerWithDeadline(cluster.API.RequestTimeout.Duration(),
httpserver.AddRequestIDs(
httpserver.LogRequests(
httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
+ Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
+ BaseContext: func(net.Listener) context.Context { return ctx },
},
Addr: listenURL.Host,
}
package httpserver
import (
+ "bufio"
"context"
+ "net"
"net/http"
"time"
requestTimeContextKey = contextKey{"requestTime"}
)
-// HandlerWithContext returns an http.Handler that changes the request
-// context to ctx (replacing http.Server's default
-// context.Background()), then calls next.
-func HandlerWithContext(ctx context.Context, next http.Handler) http.Handler {
+type hijacker interface {
+ http.ResponseWriter
+ http.Hijacker
+}
+
+// hijackNotifier wraps a ResponseWriter, calling the provided
+// Notify() func if/when the wrapped Hijacker is hijacked.
+type hijackNotifier struct {
+ hijacker
+ hijacked chan<- bool
+}
+
+func (hn hijackNotifier) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+ close(hn.hijacked)
+ return hn.hijacker.Hijack()
+}
+
+// HandlerWithDeadline cancels the request context if the request
+// takes longer than the specified timeout without having its
+// connection hijacked.
+func HandlerWithDeadline(timeout time.Duration, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ ctx, cancel := context.WithCancel(r.Context())
+ defer cancel()
+ nodeadline := make(chan bool)
+ go func() {
+ select {
+ case <-nodeadline:
+ case <-ctx.Done():
+ case <-time.After(timeout):
+ cancel()
+ }
+ }()
+ if hj, ok := w.(hijacker); ok {
+ w = hijackNotifier{hj, nodeadline}
+ }
next.ServeHTTP(w, r.WithContext(ctx))
})
}
"context"
"encoding/json"
"fmt"
+ "io/ioutil"
+ "net"
"net/http"
"net/http/httptest"
"testing"
s.ctx = ctxlog.Context(context.Background(), s.log)
}
+func (s *Suite) TestWithDeadline(c *check.C) {
+ req, err := http.NewRequest("GET", "https://foo.example/bar", nil)
+ c.Assert(err, check.IsNil)
+
+ // Short timeout cancels context in <1s
+ resp := httptest.NewRecorder()
+ HandlerWithDeadline(time.Millisecond, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ select {
+ case <-req.Context().Done():
+ w.Write([]byte("ok"))
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
+ })).ServeHTTP(resp, req.WithContext(s.ctx))
+ c.Check(resp.Body.String(), check.Equals, "ok")
+
+ // Long timeout does not cancel context in <1ms
+ resp = httptest.NewRecorder()
+ HandlerWithDeadline(time.Second, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ select {
+ case <-req.Context().Done():
+ c.Error("request context done too soon")
+ case <-time.After(time.Millisecond):
+ w.Write([]byte("ok"))
+ }
+ })).ServeHTTP(resp, req.WithContext(s.ctx))
+ c.Check(resp.Body.String(), check.Equals, "ok")
+}
+
+func (s *Suite) TestNoDeadlineAfterHijacked(c *check.C) {
+ srv := Server{
+ Addr: ":",
+ Server: http.Server{
+ Handler: HandlerWithDeadline(time.Millisecond, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ conn, _, err := w.(http.Hijacker).Hijack()
+ c.Assert(err, check.IsNil)
+ defer conn.Close()
+ select {
+ case <-req.Context().Done():
+ c.Error("request context done too soon")
+ case <-time.After(time.Second / 10):
+ conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\nok"))
+ }
+ })),
+ BaseContext: func(net.Listener) context.Context { return s.ctx },
+ },
+ }
+ srv.Start()
+ defer srv.Close()
+ resp, err := http.Get("http://" + srv.Addr)
+ c.Assert(err, check.IsNil)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), check.Equals, "ok")
+}
+
func (s *Suite) TestLogRequests(c *check.C) {
h := AddRequestIDs(LogRequests(
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
c.Assert(err, check.IsNil)
resp := httptest.NewRecorder()
- HandlerWithContext(s.ctx, h).ServeHTTP(resp, req)
+ h.ServeHTTP(resp, req.WithContext(s.ctx))
dec := json.NewDecoder(s.logdata)
c.Assert(err, check.IsNil)
resp := httptest.NewRecorder()
- HandlerWithContext(s.ctx, LogRequests(
+ LogRequests(
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(trial.statusCode)
w.Write([]byte(trial.sentBody))
}),
- )).ServeHTTP(resp, req)
+ ).ServeHTTP(resp, req.WithContext(s.ctx))
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
arvcfg.declare_config "API.MaxTokenLifetime", ActiveSupport::Duration
+arvcfg.declare_config "API.RequestTimeout", ActiveSupport::Duration
arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.set_callback :checkout, :before, ->(conn) do
+ ms = Rails.configuration.API.RequestTimeout.to_i * 1000
+ conn.execute("SET statement_timeout = #{ms}")
+ conn.execute("SET lock_timeout = #{ms}")
+end
import (
"context"
+ "net"
"net/http"
"git.arvados.org/arvados.git/sdk/go/arvados"
h := &handler{Config: srv.Config}
reg := prometheus.NewRegistry()
h.Config.Cache.registry = reg
- ctx := ctxlog.Context(context.Background(), logger)
- mh := httpserver.Instrument(reg, logger, httpserver.HandlerWithContext(ctx, httpserver.AddRequestIDs(httpserver.LogRequests(h))))
+ // Warning: when updating this to use Command() from
+ // lib/service, make sure to implement an exemption in
+ // httpserver.HandlerWithDeadline() so large file uploads are
+ // allowed to take longer than the usual API.RequestTimeout.
+ // See #13697.
+ mh := httpserver.Instrument(reg, logger, httpserver.AddRequestIDs(httpserver.LogRequests(h)))
h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler())
srv.Handler = mh
+ srv.BaseContext = func(net.Listener) context.Context { return ctxlog.Context(context.Background(), logger) }
var listen arvados.URL
for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs {
break
With no arguments, list available arvboxes.
arvopen:
- Open an Arvados uuid in web browser (http://curover.se)
+ Open an Arvados uuid in web browser (http://arvadosapi.com)
arvissue
Open an Arvados ticket in web browser (http://dev.arvados.org)
arvopen() {
if [[ -n "$1" ]] ; then
- xdg-open https://curover.se/$1
+ xdg-open https://arvadosapi.com/$1
else
echo "Open Arvados uuid in browser"
echo "Usage: arvopen uuid"
# SPDX-License-Identifier: AGPL-3.0
head=$(git log --first-parent --max-count=1 --format=%H)
-curl -X POST https://ci.curoverse.com/job/developer-run-tests/build \
- --user $(cat ~/.jenkins.ci.curoverse.com) \
+curl -X POST https://ci.arvados.org/job/developer-run-tests/build \
+ --user $(cat ~/.jenkins.ci.arvados.org) \
--data-urlencode json='{"parameter": [{"name":"git_hash", "value":"'$head'"}]}'