link
end
end
- assert_equal(['foo'], hrefs.compact.sort,
+ assert_equal(['./foo'], hrefs.compact.sort,
"download page did provide strictly file links")
click_link "foo"
assert_text "foo\nfile\n"
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.3* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
+Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.4* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
* Install PostgreSQL on the the same host as the SSO server, and dedicate that install to hosting the SSO database. This provides the best security for the SSO server, because the database does not have to accept any client connections over the network. Typical load on the SSO server is light enough that deploying both it and its database on the same host does not compromise performance.
* If you want to provide the most scalability for your Arvados cluster, install PostgreSQL for the API server on a dedicated host. This gives you the most flexibility to avoid resource contention, and tune performance separately for the API server and its database. If performance is less of a concern for your installation, you can install PostgreSQL on the API server host directly, as with the SSO server.
Find the section for your distribution below, and follow it to install PostgreSQL on each host where you will deploy it. Then follow the steps in the later section(s) to set up PostgreSQL for the Arvados service(s) that need it.
-h2. Install PostgreSQL 9.3+
+It is important to make sure that autovacuum is enabled for the PostgreSQL database that backs the API server. Autovacuum is enabled by default since PostgreSQL 8.3.
-The API server requires at least version *9.3* of PostgreSQL.
+h2. Install PostgreSQL 9.4+
+
+The API server requires at least version *9.4* of PostgreSQL.
h3(#centos7). CentOS 7
{% assign rh_version = "7" %}
h3(#debian). Debian or Ubuntu
-Debian 8 (Jessie) and Ubuntu 14.04 (Trusty) and later versions include a sufficiently recent version of Postgres.
+Debian 8 (Jessie) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
+
+Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/
# Install PostgreSQL:
<notextile><pre>~$ <span class="userinput">sudo apt-get install postgresql</span></pre></notextile>
defer os.Chdir(cwd)
chdirToPythonTests()
- bgRun(exec.Command("python", "run_test_server.py", "stop"))
+ cmd := exec.Command("python", "run_test_server.py", "stop")
+ bgRun(cmd)
+ // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
+ cmd.Wait()
}
// StartKeep starts the given number of keep servers,
chdirToPythonTests()
cmd := exec.Command("python", "run_test_server.py", "stop_keep", "--num-keep-servers", strconv.Itoa(numKeepServers))
- cmd.Stdin = nil
- cmd.Stderr = os.Stderr
- cmd.Stdout = os.Stderr
- if err := cmd.Run(); err != nil {
- log.Fatalf("%+v: %s", cmd.Args, err)
- }
+ bgRun(cmd)
+ // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
+ cmd.Wait()
}
// Start cmd, with stderr and stdout redirected to our own
<UL>
{{range .Files}}
{{if .IsDir }}
- <LI>{{" " | printf "%15s " | nbsp}}<A href="{{.Name}}/">{{.Name}}/</A></LI>
+ <LI>{{" " | printf "%15s " | nbsp}}<A href="{{print "./" .Name}}/">{{.Name}}/</A></LI>
{{else}}
- <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{.Name}}">{{.Name}}</A></LI>
+ <LI>{{.Size | printf "%15d " | nbsp}}<A href="{{print "./" .Name}}">{{.Name}}</A></LI>
{{end}}
{{end}}
</UL>
"net/http"
"net/http/httptest"
"net/url"
+ "os"
"path/filepath"
"regexp"
"strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/auth"
check "gopkg.in/check.v1"
)
}
+func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
+ s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll := arvados.Collection{ManifestText: mtxt}
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
+ c.Assert(err, check.IsNil)
+
+ u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
+}
+
// XHRs can't follow redirect-with-cookie so they rely on method=POST
// and disposition=attachment (telling us it's acceptable to respond
// with content instead of a redirect) and an Origin header that gets
} else {
c.Check(resp.Code, check.Equals, http.StatusOK)
for _, e := range trial.expect {
- c.Check(resp.Body.String(), check.Matches, `(?ms).*href="`+e+`".*`)
+ c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
}
c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
}
import stat
import tempfile
import shutil
+import errno
from functools import partial
import arvados
import StringIO
logger.info("%s passed", name)
else:
if isinstance(detail_content, StringIO.StringIO):
- sys.stderr.write(detail_content.getvalue())
+ detail_content.seek(0)
+ chunk = detail_content.read(4096)
+ while chunk:
+ try:
+ sys.stderr.write(chunk)
+ chunk = detail_content.read(4096)
+ except IOError as e:
+ if e.errno == errno.EAGAIN:
+ # try again (probably pipe buffer full)
+ pass
+ else:
+ raise
logger.info("%s failed", name)
return code
EOF = None
def __init__(self, job_uuid):
- logger.debug('load stderr events for job %s', job_uuid)
self.job_uuid = job_uuid
+ self.event_types = (['stderr'] if '-8i9sb-' in job_uuid else ['crunchstat', 'arv-mount'])
+ logger.debug('load %s events for job %s', self.event_types, self.job_uuid)
def __str__(self):
return self.job_uuid
last_id = 0
filters = [
['object_uuid', '=', self.job_uuid],
- ['event_type', '=', 'stderr']]
+ ['event_type', 'in', self.event_types]]
try:
while True:
page = arvados.api().logs().index(
projects / arvados.git / commitdiff