end
assert_text ":active/workbenchtest.git"
assert_match /git@git.*:active\/workbenchtest.git/, page.text
- assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}\/active\/workbenchtest.git/, page.text
+ assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}active\/workbenchtest.git/, page.text
end
[
COMMAND_ARR+=('-n' "$PYTHON_PKG")
COMMAND_ARR+=('-C' "build")
- if [[ -e "$WORKSPACE/$PKG_DIR/$PKG.service" ]]; then
+ systemd_unit="$WORKSPACE/$PKG_DIR/$PKG.service"
+ if [[ -e "${systemd_unit}" ]]; then
COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
fi
COMMAND_ARR+=('--depends' "$i")
done
+ # make sure the systemd service file ends up in the right place
+ # currently only used by arvados-docker-cleaner
+ if [[ -e "${systemd_unit}" ]]; then
+ COMMAND_ARR+=("usr/share/python3/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
+ fi
+
COMMAND_ARR+=("${fpm_args[@]}")
# Make sure to install all our package binaries in /usr/bin.
if p == "0" {
p = nextPort(h)
}
- cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
+ cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
svc == &cluster.Services.WebDAV ||
svc == &cluster.Services.WebDAVDownload ||
svc == &cluster.Services.Workbench1 {
- svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
} else if svc == &cluster.Services.Websocket {
- svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
}
}
if len(svc.InternalURLs) == 0 {
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
- arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
+ arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{},
}
}
}
}
if super.ClusterType == "test" {
// Add a second keepstore process.
- cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
// Create a directory-backed volume for each keepstore
// process.
code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
c.Check(stdout.String(), check.Matches, `(?ms).*TimeoutBooting: 10m\n.*`)
- c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345: {}\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345/: {}\n.*`)
}
func (s *CommandSuite) TestDump_UnknownKey(c *check.C) {
RunningJobLogRecordsToFetch: 2000
# In systems with many shared projects, loading of dashboard and topnav
- # cab be slow due to collections indexing; use the following parameters
+ # can be slow due to collections indexing; use the following parameters
# to suppress these properties
ShowRecentCollectionsOnDashboard: true
ShowUserNotifications: true
if strings.HasPrefix(host, ":") {
host = hostname + host
}
- svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+ svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host, Path: "/"}] = arvados.ServiceInstance{}
}
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
}
if client.APIHost != "" {
cluster.Services.Controller.ExternalURL.Host = client.APIHost
+ cluster.Services.Controller.ExternalURL.Path = "/"
}
if client.Scheme != "" {
cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
}
if oc.Listen != nil {
- cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.LogLevel != nil {
cluster.SystemLogs.LogLevel = *oc.LogLevel
loadOldClientConfig(cluster, oc.Client)
if oc.Listen != nil {
- cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.DefaultReplicas != nil {
cluster.Collections.DefaultReplication = *oc.DefaultReplicas
loadOldClientConfig(cluster, oc.Client)
if oc.Listen != nil {
- cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
- cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.AttachmentOnlyHost != nil {
- cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost}
+ cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost, Path: "/"}
}
if oc.ManagementToken != nil {
cluster.ManagementToken = *oc.ManagementToken
return err
}
- myURL := arvados.URL{Scheme: "http"}
+ myURL := arvados.URL{Scheme: "http", Path: "/"}
if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
myURL.Scheme = "https"
}
cluster.TLS.Key = "file://" + *v
}
if v := oc.Listen; v != nil {
- if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+ if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v, Path: "/"}]; ok {
// already listed
myURL.Host = *v
} else if len(*v) > 1 && (*v)[0] == ':' {
url := arvados.URL{
Scheme: "http",
Host: net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+ Path: "/",
}
if ks.ServiceSSLFlag {
url.Scheme = "https"
TLS: {Insecure: true}
Services:
Controller:
- ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
`, `
Clusters:
z1111:
InternalURLs:
"http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
Controller:
- ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
SystemLogs:
Format: text
LogLevel: debug
ReadOnly: true
StorageAccountName: storageacctname
StorageAccountKeyFile: `+secretkeyfile.Name()+`
- StorageBaseURL: https://example.example
+ StorageBaseURL: https://example.example/
ContainerName: testctr
LocationConstraint: true
AzureReplication: 4
}, &arvados.AzureVolumeDriverParameters{
StorageAccountName: "storageacctname",
StorageAccountKey: "secretkeydata",
- StorageBaseURL: "https://example.example",
+ StorageBaseURL: "https://example.example/",
ContainerName: "testctr",
RequestTimeout: arvados.Duration(time.Minute * 3),
ListBlobsRetryDelay: arvados.Duration(time.Minute * 4),
c.Check(v.Driver, check.Equals, expectvol.Driver)
c.Check(v.Replication, check.Equals, expectvol.Replication)
- avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+ avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345", Path: "/"}]
c.Check(ok, check.Equals, true)
c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
url := arvados.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", hostname, port),
+ Path: "/",
}
_, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
c.Check(ok, check.Equals, false)
url := arvados.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", hostname, port),
+ Path: "/",
}
_, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
c.Check(ok, check.Equals, false)
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
- _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+ _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port), Path: "/"}]
c.Check(ok, check.Equals, true)
}
c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
- c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107 does not have access to any volumes.*`)
+ c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107/ does not have access to any volumes.*`)
c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
}
cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
c.Check(err, check.IsNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
- c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com"})
+ c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
RunningJobLogRecordsToFetch: 2000
# In systems with many shared projects, loading of dashboard and topnav
- # cab be slow due to collections indexing; use the following parameters
+ # can be slow due to collections indexing; use the following parameters
# to suppress these properties
ShowRecentCollectionsOnDashboard: true
ShowUserNotifications: true
} else if url, err := url.Parse(want); err != nil {
return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
} else {
+ if url.Path == "" {
+ url.Path = "/"
+ }
return arvados.URL(*url), nil
}
import arvados.collection
from .util import collectionUUID
import ruamel.yaml as yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
if "$schemas" in workflowobj:
- sch = []
+ sch = CommentedSeq()
for s in workflowobj["$schemas"]:
sch.append(mapper.mapper(s).resolved)
workflowobj["$schemas"] = sch
u, err := url.Parse(string(text))
if err == nil {
*su = URL(*u)
+ if su.Path == "" && su.Host != "" {
+ // http://example really means http://example/
+ su.Path = "/"
+ }
}
return err
}
package arvados
import (
+ "encoding/json"
+
"github.com/ghodss/yaml"
check "gopkg.in/check.v1"
)
c.Check(itm["foo8"].IncludedScratch, check.Equals, ByteSize(0))
}
}
+
+func (s *ConfigSuite) TestURLTrailingSlash(c *check.C) {
+ var a, b map[URL]bool
+ json.Unmarshal([]byte(`{"https://foo.example": true}`), &a)
+ json.Unmarshal([]byte(`{"https://foo.example/": true}`), &b)
+ c.Check(a, check.DeepEquals, b)
+}
import (
"sync"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
)
// A ClientPool is a pool of ArvadosClients. This is useful for
// credentials. See arvados-git-httpd for an example, and sync.Pool
// for more information about garbage collection.
type ClientPool struct {
- // Initialize new clients by coping this one.
+ // Initialize new clients by copying this one.
Prototype *ArvadosClient
pool *sync.Pool
// MakeClientPool returns a new empty ClientPool, using environment
// variables to initialize the prototype.
func MakeClientPool() *ClientPool {
- proto, err := MakeArvadosClient()
+ return MakeClientPoolWith(nil)
+}
+
+// MakeClientPoolWith returns a new empty ClientPool with a previously
+// initialized arvados.Client.
+func MakeClientPoolWith(client *arvados.Client) *ClientPool {
+ var err error
+ var proto *ArvadosClient
+
+ if client == nil {
+ proto, err = MakeArvadosClient()
+ } else {
+ proto, err = New(client)
+ }
return &ClientPool{
Prototype: proto,
lastErr: err,
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
)
-// ClearCache clears the Keep service discovery cache.
+// RefreshServiceDiscovery clears the Keep service discovery cache.
func RefreshServiceDiscovery() {
var wg sync.WaitGroup
defer wg.Wait()
}
}
-// ClearCacheOnSIGHUP installs a signal handler that calls
-// ClearCache when SIGHUP is received.
+// RefreshServiceDiscoveryOnSIGHUP installs a signal handler that calls
+// RefreshServiceDiscovery when SIGHUP is received.
func RefreshServiceDiscoveryOnSIGHUP() {
svcListCacheMtx.Lock()
defer svcListCacheMtx.Unlock()
flash[:notice] = 'You have logged off'
return_to = params[:return_to] || root_url
- redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}/users/sign_out?redirect_uri=#{CGI.escape return_to}"
+ redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}users/sign_out?redirect_uri=#{CGI.escape return_to}"
end
# login - Just bounce to /auth/joshid. The only purpose of this function is
Rails.logger.warn "Copying omniauth from globals in legacy config file."
Rails.configuration.Login["SSO"]["ProviderAppID"] = APP_ID
Rails.configuration.Login["SSO"]["ProviderAppSecret"] = APP_SECRET
- Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL
+ Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL.sub(/\/$/, "") + "/"
else
Rails.application.config.middleware.use OmniAuth::Builder do
provider(:josh_id,
headers: auth(:active)
assert_response :success
json_response['items'].each do |svc|
- url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}"
+ url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}/"
assert_equal true, expect_rvz.has_key?(url), "#{url} does not match any configured service: expecting #{expect_rvz}"
rvz = expect_rvz[url]
if rvz.is_a? String
cmd = append(cmd, disp.cluster.Containers.CrunchRunArgumentsList...)
if err := disp.submit(ctr, cmd); err != nil {
var text string
- if err, ok := err.(dispatchcloud.ConstraintsNotSatisfiableError); ok {
+ switch err := err.(type) {
+ case dispatchcloud.ConstraintsNotSatisfiableError:
var logBuf bytes.Buffer
fmt.Fprintf(&logBuf, "cannot run container %s: %s\n", ctr.UUID, err)
if len(err.AvailableTypes) == 0 {
}
text = logBuf.String()
disp.UpdateState(ctr.UUID, dispatch.Cancelled)
- } else {
+ default:
text = fmt.Sprintf("Error submitting container %s to slurm: %s", ctr.UUID, err)
}
log.Print(text)
err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()})
c.Check(err, IsNil)
- c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg")
c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
}
func (h *handler) setup() {
- h.clientPool = arvadosclient.MakeClientPool()
+ // Errors will be handled at the client pool.
+ arv, _ := arvados.NewClientFromConfig(h.Config.cluster)
+ h.clientPool = arvadosclient.MakeClientPoolWith(arv)
keepclient.RefreshServiceDiscoveryOnSIGHUP()
keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
// If a config file is available, use the keepstores defined there
// instead of the legacy autodiscover mechanism via the API server
for k := range cluster.Services.Keepstore.InternalURLs {
- arv.KeepServiceURIs = append(arv.KeepServiceURIs, k.String())
+ arv.KeepServiceURIs = append(arv.KeepServiceURIs, strings.TrimRight(k.String(), "/"))
}
if cluster.SystemLogs.LogLevel == "debug" {
c.Check(err, check.IsNil)
c.Check(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.PostgreSQL.Connection, check.DeepEquals, arvados.PostgreSQLConnection{