flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
+ flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
{"WORKBENCH1", super.cluster.Services.Workbench1},
{"WS", super.cluster.Services.Websocket},
} {
- host, port, err := internalPort(cmpt.svc)
- if err != nil {
+ var host, port string
+ if len(cmpt.svc.InternalURLs) == 0 {
+ // We won't run this service, but we need an
+ // upstream port to write in our templated
+ // nginx config. Choose a port that will
+ // return 502 Bad Gateway.
+ port = "9"
+ } else if host, port, err = internalPort(cmpt.svc); err != nil {
return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
- }
- if ok, err := addrIsLocal(net.JoinHostPort(host, port)); !ok || err != nil {
- return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", host, port, err)
+ } else if ok, err := addrIsLocal(net.JoinHostPort(host, port)); !ok || err != nil {
+ return fmt.Errorf("%s addrIsLocal() failed for host %q port %q: %v", cmpt.varname, host, port, err)
}
vars[cmpt.varname+"PORT"] = port
if err != nil {
return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
- if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
- return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
+ listenAddr := net.JoinHostPort(super.ListenHost, port)
+ if ok, err := addrIsLocal(listenAddr); !ok || err != nil {
+ return fmt.Errorf("%s addrIsLocal(%q) failed: %w", cmpt.varname, listenAddr, err)
}
vars[cmpt.varname+"SSLPORT"] = port
}
ClusterType string // e.g., production
ListenHost string // e.g., localhost
ControllerAddr string // e.g., 127.0.0.1:8000
+ NoWorkbench1 bool
OwnTemporaryDatabase bool
Stderr io.Writer
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
- installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
- runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
+ if !super.NoWorkbench1 {
+ tasks = append(tasks,
+ installPassenger{src: "apps/workbench", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ )
+ }
if super.ClusterType != "test" {
tasks = append(tasks,
runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
}
}
+ if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 {
+ // When workbench1 is disabled, it gets an
+ // ExternalURL (so we have a valid listening
+ // port to write in our Nginx config) but no
+ // InternalURLs (so health checker doesn't
+ // complain).
+ continue
+ }
if len(svc.InternalURLs) == 0 {
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {},
func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend backend) {
srv := httpserver.Server{Addr: ":"}
- srv.Handler = router.New(backend, nil)
+ srv.Handler = router.New(backend, router.Config{})
c.Check(srv.Start(), check.IsNil)
s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
Scheme: "http",
})
oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.db)
- rtr := router.New(federation.New(h.Cluster), api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls))
+ rtr := router.New(federation.New(h.Cluster), router.Config{
+ MaxRequestSize: h.Cluster.API.MaxRequestSize,
+ WrapCalls: api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls),
+ })
mux.Handle("/arvados/v1/config", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
mux.Handle("/arvados/v1/collections", rtr)
tc := boot.NewTestCluster(
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
+ tc.Super.NoWorkbench1 = true
+ tc.Start()
s.testClusters[id] = tc
- s.testClusters[id].Start()
}
for _, tc := range s.testClusters {
ok := tc.WaitReady()
func (rtr *router) loadRequestParams(req *http.Request, attrsKey string) (map[string]interface{}, error) {
err := req.ParseForm()
if err != nil {
- return nil, httpError(http.StatusBadRequest, err)
+ if err.Error() == "http: request body too large" {
+ return nil, httpError(http.StatusRequestEntityTooLarge, err)
+ } else {
+ return nil, httpError(http.StatusBadRequest, err)
+ }
}
params := map[string]interface{}{}
import (
"context"
"fmt"
+ "math"
"net/http"
"strings"
)
type router struct {
- mux *mux.Router
- backend arvados.API
- wrapCalls func(api.RoutableFunc) api.RoutableFunc
+ mux *mux.Router
+ backend arvados.API
+ config Config
+}
+
+type Config struct {
+ // Return an error if request body exceeds this size. 0 means
+ // unlimited.
+ MaxRequestSize int
+
+ // If wrapCalls is not nil, it is called once for each API
+ // method, and the returned method is used in its place. This
+ // can be used to install hooks before and after each API call
+ // and alter responses; see localdb.WrapCallsInTransaction for
+ // an example.
+ WrapCalls func(api.RoutableFunc) api.RoutableFunc
}
// New returns a new router (which implements the http.Handler
// interface) that serves requests by calling Arvados API methods on
// the given backend.
-//
-// If wrapCalls is not nil, it is called once for each API method, and
-// the returned method is used in its place. This can be used to
-// install hooks before and after each API call and alter responses;
-// see localdb.WrapCallsInTransaction for an example.
-func New(backend arvados.API, wrapCalls func(api.RoutableFunc) api.RoutableFunc) *router {
+func New(backend arvados.API, config Config) *router {
rtr := &router{
- mux: mux.NewRouter(),
- backend: backend,
- wrapCalls: wrapCalls,
+ mux: mux.NewRouter(),
+ backend: backend,
+ config: config,
}
rtr.addRoutes()
return rtr
},
} {
exec := route.exec
- if rtr.wrapCalls != nil {
- exec = rtr.wrapCalls(exec)
+ if rtr.config.WrapCalls != nil {
+ exec = rtr.config.WrapCalls(exec)
}
rtr.addRoute(route.endpoint, route.defaultOpts, exec)
}
if r.Method == "OPTIONS" {
return
}
+ if r.Body != nil {
+ // Wrap r.Body in a http.MaxBytesReader(), otherwise
+ // r.ParseForm() uses a default max request body size
+ // of 10 megabytes. Note we rely on the Nginx
+ // configuration to enforce the real max body size.
+ max := int64(rtr.config.MaxRequestSize)
+ if max < 1 {
+ max = math.MaxInt64 - 1
+ }
+ r.Body = http.MaxBytesReader(w, r.Body, max)
+ }
if r.Method == "POST" {
- r.ParseForm()
+ err := r.ParseForm()
+ if err != nil {
+ if err.Error() == "http: request body too large" {
+ err = httpError(http.StatusRequestEntityTooLarge, err)
+ }
+ rtr.sendError(w, err)
+ return
+ }
if m := r.FormValue("_method"); m != "" {
r2 := *r
r = &r2
cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
url, _ := url.Parse("https://" + os.Getenv("ARVADOS_TEST_API_HOST"))
- s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider), nil)
+ s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider), Config{})
}
func (s *RouterIntegrationSuite) TearDownSuite(c *check.C) {
c.Check(jresp["kind"], check.Equals, "arvados#collection")
}
+func (s *RouterIntegrationSuite) TestMaxRequestSize(c *check.C) {
+ token := arvadostest.ActiveTokenV2
+ for _, maxRequestSize := range []int{
+ // Ensure 5M limit is enforced.
+ 5000000,
+ // Ensure 50M limit is enforced, and that a >25M body
+ // is accepted even though the default Go request size
+ // limit is 10M.
+ 50000000,
+ } {
+ s.rtr.config.MaxRequestSize = maxRequestSize
+ okstr := "a"
+ for len(okstr) < maxRequestSize/2 {
+ okstr = okstr + okstr
+ }
+
+ hdr := http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}
+
+ body := bytes.NewBufferString(url.Values{"foo_bar": {okstr}}.Encode())
+ _, rr, _ := doRequest(c, s.rtr, token, "POST", `/arvados/v1/collections`, hdr, body)
+ c.Check(rr.Code, check.Equals, http.StatusOK)
+
+ body = bytes.NewBufferString(url.Values{"foo_bar": {okstr + okstr}}.Encode())
+ _, rr, _ = doRequest(c, s.rtr, token, "POST", `/arvados/v1/collections`, hdr, body)
+ c.Check(rr.Code, check.Equals, http.StatusRequestEntityTooLarge)
+ }
+}
+
func (s *RouterIntegrationSuite) TestContainerList(c *check.C) {
token := arvadostest.ActiveTokenV2
if p.startswith("keep:") and (arvados.util.keep_locator_pattern.match(p[5:]) or
arvados.util.collection_uuid_pattern.match(p[5:])):
locator = p[5:]
- return (self.collection_cache.get(locator), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
+ rest = os.path.normpath(urllib.parse.unquote(sp[1])) if len(sp) == 2 else None
+ return (self.collection_cache.get(locator), rest)
else:
return (None, path)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+inputs: []
+outputs:
+ stuff:
+ type: Directory
+ outputBinding:
+ glob: './foo/'
+requirements:
+ ShellCommandRequirement: {}
+arguments: [{shellQuote: false, valueFrom: "mkdir -p foo && touch baz.txt && touch foo/bar.txt"}]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+inputs: []
+outputs:
+ stuff:
+ type: File
+ outputBinding:
+ glob: './foo/*.txt'
+requirements:
+ ShellCommandRequirement: {}
+arguments: [{shellQuote: false, valueFrom: "mkdir -p foo && touch baz.txt && touch foo/bar.txt"}]
output: {}
tool: wf/trick_defaults2.cwl
doc: "Test issue 17462 - secondary file objects on file defaults are not resolved"
+
+- job: null
+ output: {}
+ tool: 17521-dot-slash-glob.cwl
+ doc: "Test issue 17521 - bug with leading './' capturing files in subdirectories"
+
+- job: null
+ output: {}
+ tool: 10380-trailing-slash-dir.cwl
+ doc: "Test issue 10380 - bug with trailing slash when capturing an output directory"
"net/http"
"os"
"testing"
+ "time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
)
type ServerRequiredSuite struct{}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
RetryDelay = 0
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) SetUpTest(c *C) {
c.Assert(value, IsNil)
}
+func (s *ServerRequiredSuite) TestCreateLarge(c *C) {
+ arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
+
+ txt := arvados.SignLocator("d41d8cd98f00b204e9800998ecf8427e+0", arv.ApiToken, time.Now().Add(time.Minute), time.Minute, []byte(arvadostest.SystemRootToken))
+ // Ensure our request body is bigger than the Go http server's
+ // default max size, 10 MB.
+ for len(txt) < 12000000 {
+ txt = txt + " " + txt
+ }
+ txt = ". " + txt + " 0:0:foo\n"
+
+ resp := Dict{}
+ err = arv.Create("collections", Dict{
+ "ensure_unique_name": true,
+ "collection": Dict{
+ "is_trashed": true,
+ "name": "test",
+ "manifest_text": txt,
+ },
+ }, &resp)
+ c.Check(err, IsNil)
+ c.Check(resp["portable_data_hash"], Not(Equals), "")
+ c.Check(resp["portable_data_hash"], Not(Equals), "d41d8cd98f00b204e9800998ecf8427e+0")
+}
+
type UnitSuite struct{}
func (s *UnitSuite) TestUUIDMatch(c *C) {
server_name controller ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
+ client_max_body_size 0;
location / {
proxy_pass http://controller;
proxy_set_header Host $http_host;
}
openPath := "/" + strings.Join(targetPath, "/")
- if f, err := fs.Open(openPath); os.IsNotExist(err) {
+ f, err := fs.Open(openPath)
+ if os.IsNotExist(err) {
// Requested non-existent path
http.Error(w, notFoundMessage, http.StatusNotFound)
+ return
} else if err != nil {
// Some other (unexpected) error
http.Error(w, "open: "+err.Error(), http.StatusInternalServerError)
- } else if stat, err := f.Stat(); err != nil {
+ return
+ }
+ defer f.Close()
+ if stat, err := f.Stat(); err != nil {
// Can't get Size/IsDir (shouldn't happen with a collectionFS!)
http.Error(w, "stat: "+err.Error(), http.StatusInternalServerError)
} else if stat.IsDir() && !strings.HasSuffix(r.URL.Path, "/") {
h.serveDirectory(w, r, collection.Name, fs, openPath, true)
} else {
http.ServeContent(w, r, basename, stat.ModTime(), f)
- if wrote := int64(w.WroteBodyBytes()); wrote != stat.Size() && r.Header.Get("Range") == "" {
+ if wrote := int64(w.WroteBodyBytes()); wrote != stat.Size() && w.WroteStatus() == http.StatusOK {
// If we wrote fewer bytes than expected, it's
// too late to change the real response code
// or send an error message to the client, but
// at least we can try to put some useful
// debugging info in the logs.
n, err := f.Read(make([]byte, 1024))
- ctxlog.FromContext(r.Context()).Errorf("stat.Size()==%d but only wrote %d bytes; read(1024) returns %d, %s", stat.Size(), wrote, n, err)
-
+ ctxlog.FromContext(r.Context()).Errorf("stat.Size()==%d but only wrote %d bytes; read(1024) returns %d, %v", stat.Size(), wrote, n, err)
}
}
}
import (
"bytes"
+ "context"
"fmt"
"html"
"io/ioutil"
"path/filepath"
"regexp"
"strings"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
}
+func (s *UnitSuite) TestEmptyResponse(c *check.C) {
+ for _, trial := range []struct {
+ dataExists bool
+ sendIMSHeader bool
+ expectStatus int
+ logRegexp string
+ }{
+ // If we return no content due to a Keep read error,
+ // we should emit a log message.
+ {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
+
+ // If we return no content because the client sent an
+ // If-Modified-Since header, our response should be
+ // 304, and we should not emit a log message.
+ {true, true, http.StatusNotModified, ``},
+ } {
+ c.Logf("trial: %+v", trial)
+ arvadostest.StartKeep(2, true)
+ if trial.dataExists {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.IsNil)
+ _, _, err = kc.PutB([]byte("foo"))
+ c.Assert(err, check.IsNil)
+ }
+
+ h := handler{Config: newConfig(s.Config)}
+ u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + arvadostest.ActiveToken},
+ },
+ }
+ if trial.sendIMSHeader {
+ req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
+ }
+
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ req = req.WithContext(ctxlog.Context(context.Background(), logger))
+
+ resp := httptest.NewRecorder()
+ h.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, trial.expectStatus)
+ c.Check(resp.Body.String(), check.Equals, "")
+
+ c.Log(logbuf.String())
+ c.Check(logbuf.String(), check.Matches, trial.logRegexp)
+ }
+}
+
func (s *UnitSuite) TestInvalidUUID(c *check.C) {
bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
token := arvadostest.ActiveToken
if tok == arvadostest.ActiveToken {
c.Check(code, check.Equals, http.StatusOK)
c.Check(body, check.Equals, "foo")
-
} else {
c.Check(code >= 400, check.Equals, true)
c.Check(code < 500, check.Equals, true)
tc := boot.NewTestCluster(
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
+ tc.Super.NoWorkbench1 = true
+ tc.Start()
s.testClusters[id] = tc
- s.testClusters[id].Start()
}
for _, tc := range s.testClusters {
ok := tc.WaitReady()
projects / arvados.git / commitdiff