attrs['properties'] = {'template_uuid' => template_uuid}
attrs['priority'] = 1
attrs['state'] = "Uncommitted"
+ attrs['use_existing'] = false
# required
attrs['container_image'] = "arvados/jobs"
FileUtils.mkdir_p Rails.configuration.Workbench.RepositoryCache
[['git', 'init', '--bare', workdir],
].each do |cmd|
- system *cmd
+ system(*cmd, in: "/dev/null")
raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
end
end
'http.sslVerify',
Rails.configuration.TLS.Insecure ? 'false' : 'true'],
].each do |cmd|
- system *cmd
+ system(*cmd, in: "/dev/null")
raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
end
env = {}.
merge(ENV).
- merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token])
+ merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token],
+ 'GIT_TERMINAL_PROMPT' => '0')
cmd = ['git', '--git-dir', @workdir] + gitcmd
- io = IO.popen(env, cmd, err: [:child, :out])
+ io = IO.popen(env, cmd, err: [:child, :out], in: "/dev/null")
output = io.read
io.close
# "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs
test "browse using arv-git-http" do
repo = api_fixture('repositories')['foo']
- Repository.any_instance.
- stubs(:http_fetch_url).
- returns "#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}/#{repo['name']}.git"
commit_sha1 = '1de84a854e2b440dc53bf42f8548afa4c17da332'
visit page_with_token('active', "/repositories/#{repo['uuid']}/commit/#{commit_sha1}")
assert_text "Date: Tue Mar 18 15:55:28 2014 -0400"
"Arvados cluster cloud dispatch"
package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
"Dispatch Arvados containers to an LSF cluster"
-package_go_binary services/arv-git-httpd arvados-git-httpd "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \
"Provide authenticated http access to Arvados-hosted git repositories"
package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
"Dispatch Crunch containers on the local system"
"Make a Keep cluster accessible to clients that are not on the LAN"
package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
"Keep storage daemon, accessible to clients on the LAN"
-package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \
"Static web hosting service for user data stored in Arvados Keep"
package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
"Arvados Websocket server"
Restrict Python SDK tests to the given class
apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
Restrict Workbench tests to the given file
-services/arv-git-httpd_test="-check.vv"
+services/githttpd_test="-check.vv"
Show all log messages, even when tests pass (also works
with services/keepstore_test etc.)
ARVADOS_DEBUG=1
lib/pam
lib/service
services/api
-services/arv-git-httpd
+services/githttpd
services/crunchstat
services/dockercleaner
services/fuse
return 0
fi
. "$VENV3DIR/bin/activate"
- echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
+ echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& python3 sdk/python/tests/run_test_server.py start_keep-web \
&& checkpidfile keep-web \
&& checkhealth WebDAV \
- && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \
- && checkpidfile arv-git-httpd \
+ && python3 sdk/python/tests/run_test_server.py start_githttpd \
+ && checkpidfile githttpd \
&& checkhealth GitHTTP \
&& python3 sdk/python/tests/run_test_server.py start_ws \
&& checkpidfile ws \
. "$VENV3DIR/bin/activate" || return
cd "$WORKSPACE" \
&& python3 sdk/python/tests/run_test_server.py stop_nginx \
- && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python3 sdk/python/tests/run_test_server.py stop_githttpd \
&& python3 sdk/python/tests/run_test_server.py stop_ws \
&& python3 sdk/python/tests/run_test_server.py stop_keep-web \
&& python3 sdk/python/tests/run_test_server.py stop_keep_proxy \
)
declare -a gostuff
-gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
+gostuff=($(cd "$WORKSPACE" && git ls-files | grep '\.go$' | sed -e 's/\/[^\/]*$//' | sort -u))
install_apps/workbench() {
cd "$WORKSPACE/apps/workbench" \
do_install sdk/python pip "${VENV3DIR}/bin/"
do_install sdk/ruby
do_install services/api
- do_install services/arv-git-httpd go
do_install services/keepproxy go
do_install services/keep-web go
}
Description=Arvados git server
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/config.yml
# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
Type=notify
+EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-git-httpd
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
"git.arvados.org/arvados.git/lib/lsf"
"git.arvados.org/arvados.git/lib/recovercollection"
"git.arvados.org/arvados.git/sdk/go/health"
+ "git.arvados.org/arvados.git/services/githttpd"
+ keepweb "git.arvados.org/arvados.git/services/keep-web"
"git.arvados.org/arvados.git/services/keepproxy"
"git.arvados.org/arvados.git/services/keepstore"
"git.arvados.org/arvados.git/services/ws"
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"dispatch-lsf": lsf.DispatchCommand,
+ "git-httpd": githttpd.Command,
"install": install.Command,
"init": install.InitCommand,
+ "keep-web": keepweb.Command,
"keepproxy": keepproxy.Command,
"keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
# SPDX-License-Identifier: AGPL-3.0
[Unit]
-Description=Arvados Keep web gateway
+Description=Arvados Keep WebDAV and S3 gateway
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/config.yml
# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
[Service]
Type=notify
+EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/keep-web
# Set a reasonable default for the open file limit
LimitNOFILE=65536
- user/topics/storage-classes.html.textile.liquid
- Data Analysis with Workflows:
- user/cwl/arvados-vscode-training.html.md.liquid
+ - user/cwl/rnaseq-cwl-training.html.textile.liquid
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
The following services also support monitoring.
* API server
-* arv-git-httpd
+* arvados-git-httpd
* controller
* keep-balance
* keepproxy
table(table table-bordered table-condensed).
|_. Component|_. Description|
|api|The API server is the core of Arvados. It is backed by a Postgres database and manages information such as metadata for storage, a record of submitted compute jobs, users, groups, and associated permissions.|
-|arv-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.|
+|arvados-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.|
|arvados-dispatch-cloud|Provide elastic computing by creating and destroying cloud based virtual machines on compute demand.|
|crunch-dispatch-local|Get compute requests submitted to the API server and execute them locally.|
|crunch-dispatch-slurm|Get compute requests submitted to the API server and submit them to slurm.|
<rect class="BoundingBox" stroke="none" fill="none" x="22080" y="8491" width="4194" height="1654"/>
<path fill="rgb(114,159,207)" stroke="none" d="M 24177,10143 L 22081,10143 22081,8492 26272,8492 26272,10143 24177,10143 Z"/>
<path fill="none" stroke="rgb(52,101,164)" d="M 24177,10143 L 22081,10143 22081,8492 26272,8492 26272,10143 24177,10143 Z"/>
- <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="22852" y="9489"><tspan fill="rgb(0,0,0)" stroke="none">arv-git-httpd</tspan></tspan></tspan></text>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="494px" font-weight="400"><tspan class="TextPosition" x="22302" y="9489"><tspan fill="rgb(0,0,0)" stroke="none">arvados-git-httpd</tspan></tspan></tspan></text>
</g>
</g>
<g class="com.sun.star.drawing.CustomShape">
|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
|\3=. *User interface*|
|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, keep-web, websockets server.|
-|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).|
+|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arvados-git-httpd).|
|\3=. *Additional services*|
|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
|"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.|
h2(#config-git). Update Git Config
-Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arv-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers.
+Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arvados-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers.
<notextile>
<pre>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Getting Started with CWL"
+
+no_nav_left: true
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+notextile. <iframe src="https://doc.arvados.org/rnaseq-cwl-training" style="width:100%; height:100%; border:none" />
runPostgreSQL{},
runNginx{},
runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
- runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
+ runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP},
runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
- runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
+ runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
// fmt.Println(args[0])
// return 2
// }),
-// })("/usr/bin/multi", []string{"foobar", "baz"}))
+// })("/usr/bin/multi", []string{"foobar", "baz"}, os.Stdin, os.Stdout, os.Stderr))
//
// ...prints "baz" and exits 2.
type Multi map[string]Handler
return nil
}
var oc oldGitHttpdConfig
- err := ldr.loadOldConfigHelper("arv-git-httpd", ldr.GitHttpdPath, &oc)
+ err := ldr.loadOldConfigHelper("arvados-git-httpd", ldr.GitHttpdPath, &oc)
if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
return nil
} else if err != nil {
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
- flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+ flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arvados-git-httpd configuration `file`")
flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
for _, srcdir := range []string{
"cmd/arvados-client",
"cmd/arvados-server",
- "services/arv-git-httpd",
"services/crunch-dispatch-local",
"services/crunch-dispatch-slurm",
"services/health",
"services/keep-balance",
- "services/keep-web",
"services/keepstore",
"services/ws",
} {
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-systemd/daemon"
+ "github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
type Handler interface {
http.Handler
CheckHealth() error
+ // Done returns a channel that closes when the handler shuts
+ // itself down, or nil if this never happens.
Done() <-chan struct{}
}
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
+
+ // prog is [keepstore, keep-web, git-httpd, ...] but the
+ // legacy config flags are [-legacy-keepstore-config,
+ // -legacy-keepweb-config, -legacy-git-httpd-config, ...]
+ legacyFlag := "-legacy-" + strings.Replace(prog, "keep-", "keep", 1) + "-config"
+ args = loader.MungeLegacyConfigArgs(log, args, legacyFlag)
+
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
pprofAddr := flags.String("pprof", "", "Serve Go profile data at `[addr]:port`")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
httpserver.HandlerWithDeadline(cluster.API.RequestTimeout.Duration(),
httpserver.AddRequestIDs(
httpserver.LogRequests(
- httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))
+ interceptHealthReqs(cluster.ManagementToken, handler.CheckHealth,
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg))))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
+ Handler: ifCollectionInHost(instrumented, instrumented.ServeAPI(cluster.ManagementToken, instrumented)),
BaseContext: func(net.Listener) context.Context { return ctx },
},
Addr: listenURL.Host,
return 0
}
+// If an incoming request's target vhost has an embedded collection
+// UUID or PDH, handle it with hTrue, otherwise handle it with
+// hFalse.
+//
+// Facilitates routing "http://collections.example/metrics" to metrics
+// and "http://{uuid}.collections.example/metrics" to a file in a
+// collection.
+func ifCollectionInHost(hTrue, hFalse http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if arvados.CollectionIDFromDNSName(r.Host) != "" {
+ hTrue.ServeHTTP(w, r)
+ } else {
+ hFalse.ServeHTTP(w, r)
+ }
+ })
+}
+
+func interceptHealthReqs(mgtToken string, checkHealth func() error, next http.Handler) http.Handler {
+ mux := httprouter.New()
+ mux.Handler("GET", "/_health/ping", &health.Handler{
+ Token: mgtToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": checkHealth},
+ })
+ mux.NotFound = next
+ return ifCollectionInHost(next, mux)
+}
+
func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
"crypto/md5"
"fmt"
"regexp"
+ "strings"
"time"
"git.arvados.org/arvados.git/sdk/go/blockdigest"
)
+var (
+ UUIDMatch = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
+ PDHMatch = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+)
+
// Collection is an arvados#collection resource.
type Collection struct {
UUID string `json:"uuid"`
})
return fmt.Sprintf("%x+%d", h.Sum(nil), size)
}
+
+// CollectionIDFromDNSName returns a UUID or PDH if s begins with a
+// UUID or URL-encoded PDH; otherwise "".
+func CollectionIDFromDNSName(s string) string {
+ // Strip domain.
+ if i := strings.IndexRune(s, '.'); i >= 0 {
+ s = s[:i]
+ }
+ // Names like {uuid}--collections.example.com serve the same
+ // purpose as {uuid}.collections.example.com but can reduce
+ // cost/effort of using [additional] wildcard certificates.
+ if i := strings.Index(s, "--"); i >= 0 {
+ s = s[:i]
+ }
+ if UUIDMatch(s) {
+ return s
+ }
+ if pdh := strings.Replace(s, "-", "+", 1); PDHMatch(pdh) {
+ return pdh
+ }
+ return ""
+}
type ServiceName string
const (
- ServiceNameRailsAPI ServiceName = "arvados-api-server"
ServiceNameController ServiceName = "arvados-controller"
ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
+ ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
ServiceNameHealth ServiceName = "arvados-health"
- ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
- ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
- ServiceNameWebsocket ServiceName = "arvados-ws"
ServiceNameKeepbalance ServiceName = "keep-balance"
- ServiceNameKeepweb ServiceName = "keep-web"
ServiceNameKeepproxy ServiceName = "keepproxy"
ServiceNameKeepstore ServiceName = "keepstore"
+ ServiceNameKeepweb ServiceName = "keep-web"
+ ServiceNameRailsAPI ServiceName = "arvados-api-server"
+ ServiceNameWebsocket ServiceName = "arvados-ws"
+ ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
+ ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
)
// Map returns all services as a map, suitable for iterating over all
// services or looking up a service by name.
func (svcs Services) Map() map[ServiceName]Service {
return map[ServiceName]Service{
- ServiceNameRailsAPI: svcs.RailsAPI,
ServiceNameController: svcs.Controller,
ServiceNameDispatchCloud: svcs.DispatchCloud,
ServiceNameDispatchLSF: svcs.DispatchLSF,
+ ServiceNameGitHTTP: svcs.GitHTTP,
ServiceNameHealth: svcs.Health,
- ServiceNameWorkbench1: svcs.Workbench1,
- ServiceNameWorkbench2: svcs.Workbench2,
- ServiceNameWebsocket: svcs.Websocket,
ServiceNameKeepbalance: svcs.Keepbalance,
- ServiceNameKeepweb: svcs.WebDAV,
ServiceNameKeepproxy: svcs.Keepproxy,
ServiceNameKeepstore: svcs.Keepstore,
+ ServiceNameKeepweb: svcs.WebDAV,
+ ServiceNameRailsAPI: svcs.RailsAPI,
+ ServiceNameWebsocket: svcs.Websocket,
+ ServiceNameWorkbench1: svcs.Workbench1,
+ ServiceNameWorkbench2: svcs.Workbench2,
}
}
"net/http"
"net/url"
"os"
- "regexp"
"strings"
"sync"
"time"
type StringMatcher func(string) bool
-var UUIDMatch StringMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
-var PDHMatch StringMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+var UUIDMatch StringMatcher = arvados.UUIDMatch
+var PDHMatch StringMatcher = arvados.PDHMatch
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
&svcs.Controller,
&svcs.DispatchCloud,
&svcs.DispatchLSF,
+ &svcs.GitHTTP,
&svcs.Keepbalance,
&svcs.Keepproxy,
&svcs.Keepstore,
# If keepproxy and/or keep-web is running, send SIGHUP to make
# them discover the new keepstore services.
for svc in ('keepproxy', 'keep-web'):
- pidfile = _pidfile('keepproxy')
+ pidfile = _pidfile(svc)
if os.path.exists(pidfile):
try:
with open(pidfile) as pid:
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
logf = open(_logfilename('keepproxy'), WRITE_MODE)
kp = subprocess.Popen(
- ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
with open(_pidfile('keepproxy'), 'w') as f:
f.write(str(kp.pid))
gitport = internal_port_from_config("GitHTTP")
env = os.environ.copy()
env.pop('ARVADOS_API_TOKEN', None)
- logf = open(_logfilename('arv-git-httpd'), WRITE_MODE)
- agh = subprocess.Popen(['arv-git-httpd'],
+ logf = open(_logfilename('githttpd'), WRITE_MODE)
+ agh = subprocess.Popen(['arvados-server', 'git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
- with open(_pidfile('arv-git-httpd'), 'w') as f:
+ with open(_pidfile('githttpd'), 'w') as f:
f.write(str(agh.pid))
_wait_until_port_listens(gitport)
def stop_arv_git_httpd():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
- kill_server_pid(_pidfile('arv-git-httpd'))
+ kill_server_pid(_pidfile('githttpd'))
def run_keep_web():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
env = os.environ.copy()
logf = open(_logfilename('keep-web'), WRITE_MODE)
keepweb = subprocess.Popen(
- ['keep-web'],
+ ['arvados-server', 'keep-web'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('keep-web'), 'w') as f:
f.write(str(keepweb.pid))
keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)])
keep_web_port = find_available_port()
keep_web_external_port = find_available_port()
- keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
configsrc = os.environ.get("CONFIGSRC", None)
"WebDAVDownload": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_dl_port): {},
+ "http://%s:%s"%(localhost, keep_web_port): {},
},
},
}
'start_keep', 'stop_keep',
'start_keep_proxy', 'stop_keep_proxy',
'start_keep-web', 'stop_keep-web',
- 'start_arv-git-httpd', 'stop_arv-git-httpd',
+ 'start_githttpd', 'stop_githttpd',
'start_nginx', 'stop_nginx', 'setup_config',
]
parser = argparse.ArgumentParser()
run_keep_proxy()
elif args.action == 'stop_keep_proxy':
stop_keep_proxy()
- elif args.action == 'start_arv-git-httpd':
+ elif args.action == 'start_githttpd':
run_arv_git_httpd()
- elif args.action == 'stop_arv-git-httpd':
+ elif args.action == 'stop_githttpd':
stop_arv_git_httpd()
elif args.action == 'start_keep-web':
run_keep_web()
+++ /dev/null
-arv-git-httpd
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
-
- "git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
- "github.com/coreos/go-systemd/daemon"
- "github.com/ghodss/yaml"
- log "github.com/sirupsen/logrus"
-)
-
-var version = "dev"
-
-func main() {
- logger := log.New()
- log.SetFormatter(&log.JSONFormatter{
- TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
- })
-
- flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
- loader := config.NewLoader(os.Stdin, logger)
- loader.SetupFlags(flags)
-
- dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flags.Bool("version", false, "print version information and exit.")
-
- args := loader.MungeLegacyConfigArgs(logger, os.Args[1:], "-legacy-git-httpd-config")
- if ok, code := cmd.ParseFlags(flags, os.Args[0], args, "", os.Stderr); !ok {
- os.Exit(code)
- } else if *getVersion {
- fmt.Printf("arv-git-httpd %s\n", version)
- return
- }
-
- cfg, err := loader.Load()
- if err != nil {
- log.Fatal(err)
- }
-
- cluster, err := cfg.GetCluster("")
- if err != nil {
- log.Fatal(err)
- }
-
- if *dumpConfig {
- out, err := yaml.Marshal(cfg)
- if err != nil {
- log.Fatal(err)
- }
- _, err = os.Stdout.Write(out)
- if err != nil {
- log.Fatal(err)
- }
- return
- }
-
- srv := &server{cluster: cluster}
- if err := srv.Start(); err != nil {
- log.Fatal(err)
- }
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
- }
- log.Printf("arv-git-httpd %s started", version)
- log.Println("Listening at", srv.Addr)
- log.Println("Repository root", cluster.Git.Repositories)
- if err := srv.Wait(); err != nil {
- log.Fatal(err)
- }
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "net/http"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/health"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
-)
-
-type server struct {
- httpserver.Server
- cluster *arvados.Cluster
-}
-
-func (srv *server) Start() error {
- mux := http.NewServeMux()
- mux.Handle("/", &authHandler{handler: newGitHandler(srv.cluster), cluster: srv.cluster})
- mux.Handle("/_health/", &health.Handler{
- Token: srv.cluster.ManagementToken,
- Prefix: "/_health/",
- })
-
- var listen arvados.URL
- for listen = range srv.cluster.Services.GitHTTP.InternalURLs {
- break
- }
-
- srv.Handler = mux
- srv.Addr = listen.Host
- return srv.Server.Start()
-}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
"errors"
"os"
"regexp"
"strings"
- "sync"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/httpserver"
+ "github.com/sirupsen/logrus"
)
type authHandler struct {
handler http.Handler
clientPool *arvadosclient.ClientPool
cluster *arvados.Cluster
- setupOnce sync.Once
}
-func (h *authHandler) setup() {
- client, err := arvados.NewClientFromConfig(h.cluster)
- if err != nil {
- log.Fatal(err)
- }
-
- ac, err := arvadosclient.New(client)
- if err != nil {
- log.Fatalf("Error setting up arvados client prototype %v", err)
- }
+func (h *authHandler) CheckHealth() error {
+ return nil
+}
- h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
+func (h *authHandler) Done() <-chan struct{} {
+ return nil
}
func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
- h.setupOnce.Do(h.setup)
-
var statusCode int
var statusText string
var apiToken string
- var repoName string
- var validAPIToken bool
w := httpserver.WrapResponseWriter(wOrig)
w.Write([]byte(statusText))
}
}
-
- // If the given password is a valid token, log the first 10 characters of the token.
- // Otherwise: log the string <invalid> if a password is given, else an empty string.
- passwordToLog := ""
- if !validAPIToken {
- if len(apiToken) > 0 {
- passwordToLog = "<invalid>"
- }
- } else {
- passwordToLog = apiToken[0:10]
- }
-
- httpserver.Log(r.RemoteAddr, passwordToLog, w.WroteStatus(), statusText, repoName, r.Method, r.URL.Path)
}()
creds := auth.CredentialsFromRequest(r)
statusCode, statusText = http.StatusNotFound, "not found"
return
}
- repoName = pathParts[0]
+ repoName := pathParts[0]
repoName = strings.TrimRight(repoName, "/")
+ httpserver.SetResponseLogFields(r.Context(), logrus.Fields{
+ "repoName": repoName,
+ })
arv := h.clientPool.Get()
if arv == nil {
}
defer h.clientPool.Put(arv)
+ // Log the UUID if the supplied token is a v2 token, otherwise
+ // just the last five characters.
+ httpserver.SetResponseLogFields(r.Context(), logrus.Fields{
+ "tokenUUID": func() string {
+ if strings.HasPrefix(apiToken, "v2/") && strings.IndexRune(apiToken[3:], '/') == 27 {
+ // UUID part of v2 token
+ return apiToken[3:30]
+ } else if len(apiToken) > 5 {
+ return "[...]" + apiToken[len(apiToken)-5:]
+ } else {
+ return apiToken
+ }
+ }(),
+ })
+
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
arv.ApiToken = apiToken
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
- validAPIToken = true
if repoUUID == "" {
statusCode, statusText = http.StatusNotFound, "not found"
return
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
"io"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
}
func (s *AuthHandlerSuite) TestPermission(c *check.C) {
- h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- log.Printf("%v", r.URL)
- io.WriteString(w, r.URL.Path)
- }), cluster: s.cluster}
+ client, err := arvados.NewClientFromConfig(s.cluster)
+ c.Assert(err, check.IsNil)
+ ac, err := arvadosclient.New(client)
+ c.Assert(err, check.IsNil)
+ h := &authHandler{
+ cluster: s.cluster,
+ clientPool: &arvadosclient.ClientPool{Prototype: ac},
+ handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ log.Printf("%v", r.URL)
+ io.WriteString(w, r.URL.Path)
+ }),
+ }
baseURL, err := url.Parse("http://git.example/")
c.Assert(err, check.IsNil)
for _, trial := range []struct {
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package githttpd
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/service"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+var Command = service.Command(arvados.ServiceNameGitHTTP, newHandler)
+
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, err)
+ }
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, err)
+ }
+ return &authHandler{
+ clientPool: &arvadosclient.ClientPool{Prototype: ac},
+ cluster: cluster,
+ handler: newGitHandler(ctx, cluster),
+ }
+}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
- "log"
+ "context"
"net"
"net/http"
"net/http/cgi"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
)
// gitHandler is an http.Handler that invokes git-http-backend (or
cgi.Handler
}
-func newGitHandler(cluster *arvados.Cluster) http.Handler {
+func newGitHandler(ctx context.Context, cluster *arvados.Cluster) http.Handler {
const glBypass = "GL_BYPASS_ACCESS_CHECKS"
const glHome = "GITOLITE_HTTP_HOME"
var env []string
path = path + ":" + cluster.Git.GitoliteHome + "/bin"
} else if home, bypass := os.Getenv(glHome), os.Getenv(glBypass); home != "" || bypass != "" {
env = append(env, glHome+"="+home, glBypass+"="+bypass)
- log.Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
+ ctxlog.FromContext(ctx).Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
}
var listen arvados.URL
func (h *gitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
remoteHost, remotePort, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
- log.Printf("Internal error: SplitHostPort(r.RemoteAddr==%q): %s", r.RemoteAddr, err)
+ ctxlog.FromContext(r.Context()).Errorf("Internal error: SplitHostPort(r.RemoteAddr==%q): %s", r.RemoteAddr, err)
w.WriteHeader(http.StatusInternalServerError)
return
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
+ "context"
"net/http"
"net/http/httptest"
"net/url"
URL: u,
RemoteAddr: "[::1]:12345",
}
- h := newGitHandler(s.cluster)
+ h := newGitHandler(context.Background(), s.cluster)
h.(*gitHandler).Path = "/bin/sh"
h.(*gitHandler).Args = []string{"-c", "printf 'Content-Type: text/plain\r\n\r\n'; env"}
URL: u,
RemoteAddr: "test.bad.address.missing.port",
}
- h := newGitHandler(s.cluster)
+ h := newGitHandler(context.Background(), s.cluster)
h.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusInternalServerError)
c.Check(resp.Body.String(), check.Equals, "")
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
"io/ioutil"
var _ = check.Suite(&GitoliteSuite{})
-// GitoliteSuite tests need an API server, an arv-git-httpd server,
-// and a repository hosted by gitolite.
+// GitoliteSuite tests need an API server, an arvados-git-httpd
+// server, and a repository hosted by gitolite.
type GitoliteSuite struct {
IntegrationSuite
gitoliteHome string
func (s *GitoliteSuite) SetUpTest(c *check.C) {
var err error
- s.gitoliteHome, err = ioutil.TempDir("", "arv-git-httpd")
+ s.gitoliteHome, err = ioutil.TempDir("", "githttp")
c.Assert(err, check.Equals, nil)
runGitolite := func(prog string, args ...string) {
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
+ "context"
"errors"
"io/ioutil"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
check "gopkg.in/check.v1"
)
check.TestingT(t)
}
-// IntegrationSuite tests need an API server and an arv-git-httpd
+// IntegrationSuite tests need an API server and an arvados-git-httpd
// server. See GitSuite and GitoliteSuite.
type IntegrationSuite struct {
tmpRepoRoot string
tmpWorkdir string
- testServer *server
+ testServer *httpserver.Server
cluster *arvados.Cluster
}
var err error
if s.tmpRepoRoot == "" {
- s.tmpRepoRoot, err = ioutil.TempDir("", "arv-git-httpd")
+ s.tmpRepoRoot, err = ioutil.TempDir("", "githttp")
c.Assert(err, check.Equals, nil)
}
- s.tmpWorkdir, err = ioutil.TempDir("", "arv-git-httpd")
+ s.tmpWorkdir, err = ioutil.TempDir("", "githttp")
c.Assert(err, check.Equals, nil)
_, err = exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-382brsig8rp3666.git").Output()
c.Assert(err, check.Equals, nil)
s.cluster.ManagementToken = arvadostest.ManagementToken
}
- s.testServer = &server{cluster: s.cluster}
+ s.testServer = &httpserver.Server{}
+ s.testServer.Handler = httpserver.LogRequests(newHandler(context.Background(), s.cluster, "", nil))
err = s.testServer.Start()
c.Assert(err, check.Equals, nil)
c.Assert(err, check.Equals, nil)
// Clear ARVADOS_API_* env vars before starting up the server,
- // to make sure arv-git-httpd doesn't use them or complain
+ // to make sure arvados-git-httpd doesn't use them or complain
// about them being missing.
os.Unsetenv("ARVADOS_API_HOST")
os.Unsetenv("ARVADOS_API_HOST_INSECURE")
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package githttpd
import (
- "net/http"
- "net/http/httptest"
"os"
"os/exec"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
-
check "gopkg.in/check.v1"
)
c.Log(string(msg))
c.Assert(err, check.Equals, nil)
}
-
-func (s *GitSuite) TestHealthCheckPing(c *check.C) {
- req, err := http.NewRequest("GET",
- "http://"+s.testServer.Addr+"/_health/ping",
- nil)
- c.Assert(err, check.Equals, nil)
- req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
-
- resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, 200)
- c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
-}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"sync"
type cache struct {
cluster *arvados.Cluster
- config *arvados.WebDAVCacheConfig // TODO: use cluster.Collections.WebDAV instead
logger logrus.FieldLogger
registry *prometheus.Registry
metrics cacheMetrics
func (c *cache) setup() {
var err error
- c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries)
+ c.pdhs, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxUUIDEntries)
if err != nil {
panic(err)
}
- c.collections, err = lru.New2Q(c.config.MaxCollectionEntries)
+ c.collections, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxCollectionEntries)
if err != nil {
panic(err)
}
- c.sessions, err = lru.New2Q(c.config.MaxSessions)
+ c.sessions, err = lru.New2Q(c.cluster.Collections.WebDAVCache.MaxSessions)
if err != nil {
panic(err)
}
return err
}
c.collections.Add(client.AuthToken+"\000"+updated.PortableDataHash, &cachedCollection{
- expire: time.Now().Add(time.Duration(c.config.TTL)),
+ expire: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
collection: &updated,
})
c.pdhs.Add(coll.UUID, &cachedPDH{
- expire: time.Now().Add(time.Duration(c.config.TTL)),
- refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+ expire: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL)),
+ refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
pdh: updated.PortableDataHash,
})
return nil
if sess == nil {
c.metrics.sessionMisses.Inc()
sess = &cachedSession{
- expire: now.Add(c.config.TTL.Duration()),
+ expire: now.Add(c.cluster.Collections.WebDAVCache.TTL.Duration()),
}
var err error
sess.client, err = arvados.NewClientFromConfig(c.cluster)
return nil, err
}
c.logger.Debugf("cache(%s): retrieved manifest, caching with pdh %s", targetID, retrieved.PortableDataHash)
- exp := time.Now().Add(time.Duration(c.config.TTL))
+ exp := time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.TTL))
if targetID != retrieved.PortableDataHash {
c.pdhs.Add(targetID, &cachedPDH{
expire: exp,
- refresh: time.Now().Add(time.Duration(c.config.UUIDTTL)),
+ refresh: time.Now().Add(time.Duration(c.cluster.Collections.WebDAVCache.UUIDTTL)),
pdh: retrieved.PortableDataHash,
})
}
expire: exp,
collection: &retrieved,
})
- if int64(len(retrieved.ManifestText)) > c.config.MaxCollectionBytes/int64(c.config.MaxCollectionEntries) {
+ if int64(len(retrieved.ManifestText)) > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/int64(c.cluster.Collections.WebDAVCache.MaxCollectionEntries) {
select {
case c.chPruneCollections <- struct{}{}:
default:
}
}
for i, k := range keys {
- if size <= c.config.MaxCollectionBytes/2 {
+ if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
break
}
if expired[i] {
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
// Hit the same collection 5 times using the same token. Only
// the first req should cause an API call; the next 4 should
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollectionPDH, forceReload)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- cache := newConfig(ctxlog.TestLogger(c), s.Config).Cache
- cache.registry = prometheus.NewRegistry()
+ cache := &cache{
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ registry: prometheus.NewRegistry(),
+ }
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
}
func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc func(arvados.Collection) (string, string, string), skip func(string) bool) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
-
testdata := []byte("the human tragedy consists in the necessity of living with the consequences of actions performed under the pressure of compulsions we do not understand")
tempdir, err := ioutil.TempDir("", "keep-web-test-")
match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
},
} {
- c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
+ c.Logf("%s %+v", s.testServer.URL, trial)
if skip != nil && skip(trial.path) {
c.Log("(skip)")
continue
c.Assert(err, check.IsNil)
defer os.RemoveAll(tempdir)
- cmd := exec.Command("cadaver", "http://"+s.testServer.Addr+path)
+ cmd := exec.Command("cadaver", s.testServer.URL+path)
if password != "" {
// cadaver won't try username/password authentication
// unless the server responds 401 to an
// unauthenticated request, which it only does in
// AttachmentOnlyHost, TrustAllContent, and
// per-collection vhost cases.
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.URL[7:]
cmd.Env = append(os.Environ(), "HOME="+tempdir)
f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
//
// Proxy configuration
//
-// Keep-web does not support TLS natively. Typically, it is installed
-// behind a proxy like nginx.
+// Typically, keep-web is installed behind a proxy like nginx.
//
// Here is an example nginx configuration.
//
// }
//
// It is not necessary to run keep-web on the same host as the nginx
-// proxy. However, TLS is not used between nginx and keep-web, so
+// proxy. However, if TLS is not used between nginx and keep-web, the
// intervening networks must be secured by other means.
//
// Anonymous downloads
// /metrics. The same information is also available as JSON at
// /metrics.json.
//
-package main
+package keepweb
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"encoding/json"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
type handler struct {
- Config *Config
- MetricsAPI http.Handler
- clientPool *arvadosclient.ClientPool
- setupOnce sync.Once
- healthHandler http.Handler
- webdavLS webdav.LockSystem
-}
-
-// parseCollectionIDFromDNSName returns a UUID or PDH if s begins with
-// a UUID or URL-encoded PDH; otherwise "".
-func parseCollectionIDFromDNSName(s string) string {
- // Strip domain.
- if i := strings.IndexRune(s, '.'); i >= 0 {
- s = s[:i]
- }
- // Names like {uuid}--collections.example.com serve the same
- // purpose as {uuid}.collections.example.com but can reduce
- // cost/effort of using [additional] wildcard certificates.
- if i := strings.Index(s, "--"); i >= 0 {
- s = s[:i]
- }
- if arvadosclient.UUIDMatch(s) {
- return s
- }
- if pdh := strings.Replace(s, "-", "+", 1); arvadosclient.PDHMatch(pdh) {
- return pdh
- }
- return ""
+ Cache cache
+ Cluster *arvados.Cluster
+ clientPool *arvadosclient.ClientPool
+ setupOnce sync.Once
+ webdavLS webdav.LockSystem
}
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
func (h *handler) setup() {
// Errors will be handled at the client pool.
- arv, _ := arvados.NewClientFromConfig(h.Config.cluster)
+ arv, _ := arvados.NewClientFromConfig(h.Cluster)
h.clientPool = arvadosclient.MakeClientPoolWith(arv)
- keepclient.RefreshServiceDiscoveryOnSIGHUP()
- keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
-
- h.healthHandler = &health.Handler{
- Token: h.Config.cluster.ManagementToken,
- Prefix: "/_health/",
- }
+ keepclient.DefaultBlockCache.MaxBlocks = h.Cluster.Collections.WebDAVCache.MaxBlockEntries
// Even though we don't accept LOCK requests, every webdav
// handler must have a non-nil LockSystem.
}
}
+// CheckHealth implements service.Handler.
+func (h *handler) CheckHealth() error {
+ return nil
+}
+
+// Done implements service.Handler.
+func (h *handler) Done() <-chan struct{} {
+ return nil
+}
+
// ServeHTTP implements http.Handler.
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h.setupOnce.Do(h.setup)
w := httpserver.WrapResponseWriter(wOrig)
- if strings.HasPrefix(r.URL.Path, "/_health/") && r.Method == "GET" {
- h.healthHandler.ServeHTTP(w, r)
- return
- }
-
if method := r.Header.Get("Access-Control-Request-Method"); method != "" && r.Method == "OPTIONS" {
if !browserMethod[method] && !webdavMethod[method] {
w.WriteHeader(http.StatusMethodNotAllowed)
var pathToken bool
var attachment bool
var useSiteFS bool
- credentialsOK := h.Config.cluster.Collections.TrustAllContent
+ credentialsOK := h.Cluster.Collections.TrustAllContent
reasonNotAcceptingCredentials := ""
- if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Config.cluster.Services.WebDAVDownload.ExternalURL.Host) {
+ if r.Host != "" && stripDefaultPort(r.Host) == stripDefaultPort(h.Cluster.Services.WebDAVDownload.ExternalURL.Host) {
credentialsOK = true
attachment = true
} else if r.FormValue("disposition") == "attachment" {
if !credentialsOK {
reasonNotAcceptingCredentials = fmt.Sprintf("vhost %q does not specify a single collection ID or match Services.WebDAVDownload.ExternalURL %q, and Collections.TrustAllContent is false",
- r.Host, h.Config.cluster.Services.WebDAVDownload.ExternalURL)
+ r.Host, h.Cluster.Services.WebDAVDownload.ExternalURL)
}
- if collectionID = parseCollectionIDFromDNSName(r.Host); collectionID != "" {
+ if collectionID = arvados.CollectionIDFromDNSName(r.Host); collectionID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
} else if r.URL.Path == "/status.json" {
h.serveStatus(w, r)
return
- } else if strings.HasPrefix(r.URL.Path, "/metrics") {
- h.MetricsAPI.ServeHTTP(w, r)
- return
} else if siteFSDir[pathParts[0]] {
useSiteFS = true
} else if len(pathParts) >= 1 && strings.HasPrefix(pathParts[0], "c=") {
if tokens == nil {
tokens = reqTokens
- if h.Config.cluster.Users.AnonymousUserToken != "" {
- tokens = append(tokens, h.Config.cluster.Users.AnonymousUserToken)
+ if h.Cluster.Users.AnonymousUserToken != "" {
+ tokens = append(tokens, h.Cluster.Users.AnonymousUserToken)
}
}
tokenResult := make(map[string]int)
for _, arv.ApiToken = range tokens {
var err error
- collection, err = h.Config.Cache.Get(arv, collectionID, forceReload)
+ collection, err = h.Cache.Get(arv, collectionID, forceReload)
if err == nil {
// Success
break
}
// Check configured permission
- _, sess, err := h.Config.Cache.GetSession(arv.ApiToken)
- tokenUser, err = h.Config.Cache.GetTokenUser(arv.ApiToken)
+ _, sess, err := h.Cache.GetSession(arv.ApiToken)
+ tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
if webdavMethod[r.Method] {
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
ResponseWriter: w,
logger: ctxlog.FromContext(r.Context()),
update: func() error {
- return h.Config.Cache.Update(client, *collection, writefs)
+ return h.Cache.Update(client, *collection, writefs)
}}
}
h := webdav.Handler{
return
}
- fs, sess, err := h.Config.Cache.GetSession(tokens[0])
+ fs, sess, err := h.Cache.GetSession(tokens[0])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
+ fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution)
f, err := fs.Open(r.URL.Path)
if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
- tokenUser, err := h.Config.Cache.GetTokenUser(tokens[0])
+ tokenUser, err := h.Cache.GetTokenUser(tokens[0])
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return
var permitDownload bool
var permitUpload bool
if tokenUser != nil && tokenUser.IsAdmin {
- permitUpload = h.Config.cluster.Collections.WebDAVPermission.Admin.Upload
- permitDownload = h.Config.cluster.Collections.WebDAVPermission.Admin.Download
+ permitUpload = h.Cluster.Collections.WebDAVPermission.Admin.Upload
+ permitDownload = h.Cluster.Collections.WebDAVPermission.Admin.Download
} else {
- permitUpload = h.Config.cluster.Collections.WebDAVPermission.User.Upload
- permitDownload = h.Config.cluster.Collections.WebDAVPermission.User.Download
+ permitUpload = h.Cluster.Collections.WebDAVPermission.User.Upload
+ permitDownload = h.Cluster.Collections.WebDAVPermission.User.Download
}
if (method == "PUT" || method == "POST") && !permitUpload {
// Disallow operations that upload new files.
WithField("user_full_name", user.FullName)
useruuid = user.UUID
} else {
- useruuid = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Config.cluster.ClusterID)
+ useruuid = fmt.Sprintf("%s-tpzed-anonymouspublic", h.Cluster.ClusterID)
}
if collection == nil && fs != nil {
collection, filepath = h.determineCollection(fs, filepath)
}
if r.Method == "PUT" || r.Method == "POST" {
log.Info("File upload")
- if h.Config.cluster.Collections.WebDAVLogEvents {
+ if h.Cluster.Collections.WebDAVLogEvents {
go func() {
lr := arvadosclient.Dict{"log": arvadosclient.Dict{
"object_uuid": useruuid,
props["portable_data_hash"] = collection.PortableDataHash
}
log.Info("File download")
- if h.Config.cluster.Collections.WebDAVLogEvents {
+ if h.Cluster.Collections.WebDAVLogEvents {
go func() {
lr := arvadosclient.Dict{"log": arvadosclient.Dict{
"object_uuid": useruuid,
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
}
type UnitSuite struct {
- Config *arvados.Config
+ cluster *arvados.Cluster
+ handler *handler
}
func (s *UnitSuite) SetUpTest(c *check.C) {
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
+ logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
ldr.Path = "-"
cfg, err := ldr.Load()
c.Assert(err, check.IsNil)
- s.Config = cfg
+ cc, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.cluster = cc
+ s.handler = &handler{
+ Cluster: cc,
+ Cache: cache{
+ cluster: cc,
+ logger: logger,
+ registry: prometheus.NewRegistry(),
+ },
+ }
}
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
+ h := s.handler
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
req := &http.Request{
Method: "OPTIONS",
c.Assert(err, check.IsNil)
}
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
req := &http.Request{
Method: "GET",
req = req.WithContext(ctxlog.Context(context.Background(), logger))
resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, trial.expectStatus)
c.Check(resp.Body.String(), check.Equals, "")
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- cfg := newConfig(ctxlog.TestLogger(c), s.Config)
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- h := handler{Config: cfg}
- h.ServeHTTP(resp, req)
+ s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
}
}
URL: u,
RequestURI: u.RequestURI(),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
for _, port := range []string{"80", "443", "8000"} {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
req := &http.Request{
Method: "GET",
func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return req, resp
}
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
resp := s.testVhostRedirectTokenToCookie(c, "GET",
"download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
- s.testServer.Config.cluster.Collections.TrustAllContent = true
+ s.handler.Cluster.Collections.TrustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
"?api_token="+arvadostest.ActiveToken,
}
func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.FooCollection+"/foo",
}
func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
}
func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
+ s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
s.testVhostRedirectTokenToCookie(c, "GET",
"example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
"",
}
func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\"odd' path chars.*`)
}
func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
arv := arvados.NewClientFromEnv()
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
name := "foo/bar/baz"
nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
+ fs, err := (&arvados.Collection{}).FileSystem(client, nil)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
c.Assert(err, check.IsNil)
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, expectRegexp)
}
}.Encode())),
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
},
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
c.Check(resp.Body.String(), check.Equals, expectRespBody)
}()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
return resp
}
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Header().Get("Location"), check.Equals, "")
return resp
}
func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+ s.handler.Cluster.Users.AnonymousUserToken = ""
s.testDirectoryListing(c)
}
func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
authHeader := http.Header{
"Authorization": {"OAuth2 " + arvadostest.ActiveToken},
}
RequestURI: u.RequestURI(),
Header: copyHeader(trial.header),
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
for resp.Code == http.StatusSeeOther {
u, _ := req.URL.Parse(resp.Header().Get("Location"))
req.AddCookie(c)
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
}
if trial.redirect != "" {
c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
}
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expect == nil {
- if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
+ if s.handler.Cluster.Users.AnonymousUserToken == "" {
c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
} else {
c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
var updated arvados.Collection
for _, fnm := range []string{"foo.txt", "bar.txt"} {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
req := &http.Request{
Method: "DELETE",
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNoContent)
updated = arvados.Collection{}
c.Check(updated.ManifestText, check.Equals, "")
}
-func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
- s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
- authHeader := http.Header{
- "Authorization": {"Bearer " + arvadostest.ManagementToken},
- }
-
- resp := httptest.NewRecorder()
- u := mustParseURL("http://download.example.com/_health/ping")
- req := &http.Request{
- Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
- Header: authHeader,
- }
- s.testServer.Handler.ServeHTTP(resp, req)
-
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
-}
-
func (s *IntegrationSuite) TestFileContentType(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+ s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.ActiveToken
- arv, err := arvadosclient.New(&client)
+ arv, err := arvadosclient.New(client)
c.Assert(err, check.Equals, nil)
kc, err := keepclient.MakeKeepClient(arv)
c.Assert(err, check.Equals, nil)
- fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+ fs, err := (&arvados.Collection{}).FileSystem(client, kc)
c.Assert(err, check.IsNil)
trials := []struct {
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
c.Check(resp.Body.String(), check.Equals, trial.content)
}
func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
- s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
req := &http.Request{
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
}
req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
req.Host = req.URL.Host
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, status)
}
reqPut.Host = req.URL.Host
reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, &reqPut)
+ s.handler.ServeHTTP(resp, &reqPut)
c.Check(resp.Code, check.Equals, http.StatusCreated)
// new file should not appear in colls[1]
return hc
}
-func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, req *http.Request,
+func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
- client := s.testServer.Config.Client
+ client := arvados.NewClientFromEnv()
client.AuthToken = arvadostest.AdminToken
var logentries arvados.LogList
limit1 := 1
logger.Out = &logbuf
resp := httptest.NewRecorder()
req = req.WithContext(ctxlog.Context(context.Background(), logger))
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if perm {
c.Check(resp.Result().StatusCode, check.Equals, successCode)
}
func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
- config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
- h := handler{Config: config}
u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
- config.cluster.Collections.TrustAllContent = true
+ s.handler.Cluster.Collections.TrustAllContent = true
for _, adminperm := range []bool{true, false} {
for _, userperm := range []bool{true, false} {
- config.cluster.Collections.WebDAVPermission.Admin.Download = adminperm
- config.cluster.Collections.WebDAVPermission.User.Download = userperm
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
// Test admin permission
req := &http.Request{
"Authorization": {"Bearer " + arvadostest.AdminToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", adminperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
// Test user permission
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", userperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
}
}
- config.cluster.Collections.WebDAVPermission.User.Download = true
+ s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
"http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
}
"Authorization": {"Bearer " + arvadostest.ActiveToken},
},
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
+ s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
}
func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
- config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
- h := handler{Config: config}
-
for _, adminperm := range []bool{true, false} {
for _, userperm := range []bool{true, false} {
- arv := s.testServer.Config.Client
+ arv := arvados.NewClientFromEnv()
arv.AuthToken = arvadostest.ActiveToken
var coll arvados.Collection
u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
- config.cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
- config.cluster.Collections.WebDAVPermission.User.Upload = userperm
+ s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
+ s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
// Test admin permission
req := &http.Request{
},
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", adminperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
arvadostest.AdminUserUUID, coll.UUID, "bar")
// Test user permission
},
Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
}
- s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", userperm,
+ s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
arvadostest.ActiveUserUUID, coll.UUID, "bar")
}
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"context"
- "flag"
- "fmt"
"mime"
"os"
- "git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/coreos/go-systemd/daemon"
- "github.com/ghodss/yaml"
- "github.com/sirupsen/logrus"
- log "github.com/sirupsen/logrus"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
)
var (
version = "dev"
)
-// Config specifies server configuration.
-type Config struct {
- Client arvados.Client
- Cache cache
- cluster *arvados.Cluster
-}
-
-func newConfig(logger logrus.FieldLogger, arvCfg *arvados.Config) *Config {
- cfg := Config{}
- var cls *arvados.Cluster
- var err error
- if cls, err = arvCfg.GetCluster(""); err != nil {
- log.Fatal(err)
- }
- cfg.cluster = cls
- cfg.Cache.config = &cfg.cluster.Collections.WebDAVCache
- cfg.Cache.cluster = cls
- cfg.Cache.logger = logger
- return &cfg
-}
-
-func init() {
- // MakeArvadosClient returns an error if this env var isn't
- // available as a default token (even if we explicitly set a
- // different token before doing anything with the client). We
- // set this dummy value during init so it doesn't clobber the
- // one used by "run test servers".
- if os.Getenv("ARVADOS_API_TOKEN") == "" {
- os.Setenv("ARVADOS_API_TOKEN", "xxx")
- }
+var Command = service.Command(arvados.ServiceNameKeepweb, newHandlerOrErrorHandler)
- log.SetFormatter(&log.JSONFormatter{
- TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
- })
-}
-
-func configure(logger log.FieldLogger, args []string) (*Config, error) {
- flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
-
- loader := config.NewLoader(os.Stdin, logger)
- loader.SetupFlags(flags)
-
- dumpConfig := flags.Bool("dump-config", false,
- "write current configuration to stdout and exit")
- getVersion := flags.Bool("version", false,
- "print version information and exit.")
-
- prog := args[0]
- args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepweb-config")
- if ok, code := cmd.ParseFlags(flags, prog, args, "", os.Stderr); !ok {
- os.Exit(code)
- } else if *getVersion {
- fmt.Printf("%s %s\n", args[0], version)
- return nil, nil
- }
-
- arvCfg, err := loader.Load()
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ h, err := newHandler(ctx, cluster, token, reg)
if err != nil {
- return nil, err
- }
- cfg := newConfig(logger, arvCfg)
-
- if *dumpConfig {
- out, err := yaml.Marshal(cfg)
- if err != nil {
- return nil, err
- }
- _, err = os.Stdout.Write(out)
- return nil, err
+ return service.ErrorHandler(ctx, cluster, err)
}
- return cfg, nil
+ return h
}
-func main() {
- initLogger := log.StandardLogger()
- logger := initLogger.WithField("PID", os.Getpid())
- cfg, err := configure(logger, os.Args)
- if err != nil {
- log.Fatal(err)
- } else if cfg == nil {
- return
- }
- logger = logger.WithField("ClusterID", cfg.cluster.ClusterID)
- logger.Printf("keep-web %s started", version)
- ctx := ctxlog.Context(context.Background(), logger)
-
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) (*handler, error) {
+ logger := ctxlog.FromContext(ctx)
if ext := ".txt"; mime.TypeByExtension(ext) == "" {
- log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
- }
-
- os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
- srv := &server{Config: cfg}
- if err := srv.Start(ctx, initLogger); err != nil {
- logger.Fatal(err)
- }
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- logger.Printf("Error notifying init daemon: %v", err)
- }
- logger.Println("Listening at", srv.Addr)
- if err := srv.Wait(); err != nil {
- logger.Fatal(err)
- }
+ logger.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
+ }
+
+ keepclient.RefreshServiceDiscoveryOnSIGHUP()
+ os.Setenv("ARVADOS_API_HOST", cluster.Services.Controller.ExternalURL.Host)
+ return &handler{
+ Cluster: cluster,
+ Cache: cache{
+ cluster: cluster,
+ logger: logger,
+ registry: reg,
+ },
+ }, nil
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"fmt"
"Range": {"bytes=" + trial.header},
},
}
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
if trial.expectObey {
c.Check(resp.Code, check.Equals, http.StatusPartialContent)
c.Check(resp.Body.Len(), check.Equals, len(trial.expectBody))
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"crypto/hmac"
}
client := (&arvados.Client{
- APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host,
- Insecure: h.Config.cluster.TLS.Insecure,
+ APIHost: h.Cluster.Services.Controller.ExternalURL.Host,
+ Insecure: h.Cluster.TLS.Insecure,
}).WithRequestID(r.Header.Get("X-Request-Id"))
var aca arvados.APIClientAuthorization
var secret string
if len(key) == 27 && key[5:12] == "-gj3su-" {
// Access key is the UUID of an Arvados token, secret
// key is the secret part.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Cluster.SystemRootToken)
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil)
secret = aca.APIToken
} else {
// Use a single session (cached FileSystem) across
// multiple read requests.
var sess *cachedSession
- fs, sess, err = h.Config.Cache.GetSession(token)
+ fs, sess, err = h.Cache.GetSession(token)
if err != nil {
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
defer release()
fs = client.SiteFileSystem(kc)
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
+ fs.ForwardSlashNameSubstitution(h.Cluster.Collections.ForwardSlashNameSubstitution)
}
var objectNameGiven bool
var bucketName string
fspath := "/by_id"
- if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ if id := arvados.CollectionIDFromDNSName(r.Host); id != "" {
fspath += "/" + id
bucketName = id
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
fmt.Fprintln(w, `<LocationConstraint><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
- h.Config.cluster.ClusterID+
+ h.Cluster.ClusterID+
`</LocationConstraint></LocationConstraint>`)
} else if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
// GetBucketWebsite ("GET /bucketid/?website"), GetBucketTagging, etc.
}
return true
}
- if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Config.cluster.Collections.S3FolderObjects {
+ if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Cluster.Collections.S3FolderObjects {
w.Header().Set("Content-Type", "application/x-directory")
w.WriteHeader(http.StatusOK)
return true
return true
}
- tokenUser, err := h.Config.Cache.GetTokenUser(token)
+ tokenUser, err := h.Cache.GetTokenUser(token)
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
}
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
- if !h.Config.cluster.Collections.S3FolderObjects {
+ if !h.Cluster.Collections.S3FolderObjects {
s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest)
return true
}
}
defer f.Close()
- tokenUser, err := h.Config.Cache.GetTokenUser(token)
+ tokenUser, err := h.Cache.GetTokenUser(token)
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
http.Error(w, "Not permitted", http.StatusForbidden)
return true
return true
}
// Ensure a subsequent read operation will see the changes.
- h.Config.Cache.ResetSession(token)
+ h.Cache.ResetSession(token)
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
return true
}
// Ensure a subsequent read operation will see the changes.
- h.Config.Cache.ResetSession(token)
+ h.Cache.ResetSession(token)
w.WriteHeader(http.StatusNoContent)
return true
default:
if path < params.marker || path < params.prefix || path <= params.startAfter {
return nil
}
- if fi.IsDir() && !h.Config.cluster.Collections.S3FolderObjects {
+ if fi.IsDir() && !h.Cluster.Collections.S3FolderObjects {
// Note we don't add anything to
// commonPrefixes here even if delimiter is
// "/". We descend into the directory, and
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
region := aws.Region{
Name: "zzzzz",
- S3Endpoint: "http://" + s.testServer.Addr,
+ S3Endpoint: s.testServer.URL,
}
client := s3.New(*auth, region)
client.Signature = aws.V4Signature
c.Check(err, check.IsNil)
rdr, err := bucket.GetReader(objname)
- if strings.HasSuffix(trial.path, "/") && !s.testServer.Config.cluster.Collections.S3FolderObjects {
+ if strings.HasSuffix(trial.path, "/") && !s.handler.Cluster.Collections.S3FolderObjects {
c.Check(err, check.NotNil)
continue
} else if !c.Check(err, check.IsNil) {
s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/")
}
func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) {
- s.testServer.Config.cluster.Collections.S3FolderObjects = true
+ s.handler.Cluster.Collections.S3FolderObjects = true
for _, trial := range []struct {
path string
}{
s.testS3PutObjectFailure(c, stage.projbucket, stage.coll.Name+"/")
}
func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
- s.testServer.Config.cluster.Collections.S3FolderObjects = false
+ s.handler.Cluster.Collections.S3FolderObjects = false
var wg sync.WaitGroup
for _, trial := range []struct {
c.Assert(err, check.IsNil)
s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
rr := httptest.NewRecorder()
- s.testServer.Server.Handler.ServeHTTP(rr, req)
+ s.handler.ServeHTTP(rr, req)
resp := rr.Result()
c.Check(resp.StatusCode, check.Equals, trial.responseCode)
body, err := ioutil.ReadAll(resp.Body)
defer stage.teardown(c)
var markers int
- for markers, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
+ for markers, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
dirs := 2
filesPerDir := 1001
stage.writeBigDirs(c, dirs, filesPerDir)
}
}
func (s *IntegrationSuite) testS3List(c *check.C, bucket *s3.Bucket, prefix string, pageSize, expectFiles int) {
- c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.testServer.Config.cluster.Collections.S3FolderObjects)
+ c.Logf("testS3List: prefix=%q pageSize=%d S3FolderObjects=%v", prefix, pageSize, s.handler.Cluster.Collections.S3FolderObjects)
expectPageSize := pageSize
if expectPageSize > 1000 {
expectPageSize = 1000
}
func (s *IntegrationSuite) TestS3CollectionListRollup(c *check.C) {
- for _, s.testServer.Config.cluster.Collections.S3FolderObjects = range []bool{false, true} {
+ for _, s.handler.Cluster.Collections.S3FolderObjects = range []bool{false, true} {
s.testS3CollectionListRollup(c)
}
}
}
}
markers := 0
- if s.testServer.Config.cluster.Collections.S3FolderObjects {
+ if s.handler.Cluster.Collections.S3FolderObjects {
markers = 1
}
c.Check(allfiles, check.HasLen, dirs*(filesPerDir+markers)+3+markers)
sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
Region: aws_aws.String("auto"),
- Endpoint: aws_aws.String("http://" + s.testServer.Addr),
+ Endpoint: aws_aws.String(s.testServer.URL),
Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
S3ForcePathStyle: aws_aws.Bool(true),
}))
sess := aws_session.Must(aws_session.NewSession(&aws_aws.Config{
Region: aws_aws.String("auto"),
- Endpoint: aws_aws.String("http://" + s.testServer.Addr),
+ Endpoint: aws_aws.String(s.testServer.URL),
Credentials: aws_credentials.NewStaticCredentials(url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2), ""),
S3ForcePathStyle: aws_aws.Bool(true),
}))
stage := s.s3setup(c)
defer stage.teardown(c)
- cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
+ cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
buf, err := cmd.CombinedOutput()
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
// keep-web's signature verification wrt chars like "|"
// (neither reserved nor unreserved) and "," (not normally
// percent-encoded in a path).
- cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar")
+ tmpfile := c.MkDir() + "/dstfile"
+ cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
buf, err = cmd.CombinedOutput()
c.Check(err, check.NotNil)
c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
if service == "s3" {
return aws.Endpoint{
- URL: "http://" + s.testServer.Addr,
+ URL: s.testServer.URL,
SigningRegion: "custom-signing-region",
}, nil
}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "context"
- "net"
- "net/http"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
-)
-
-type server struct {
- httpserver.Server
- Config *Config
-}
-
-func (srv *server) Start(ctx context.Context, logger *logrus.Logger) error {
- h := &handler{Config: srv.Config}
- reg := prometheus.NewRegistry()
- h.Config.Cache.registry = reg
- // Warning: when updating this to use Command() from
- // lib/service, make sure to implement an exemption in
- // httpserver.HandlerWithDeadline() so large file uploads are
- // allowed to take longer than the usual API.RequestTimeout.
- // See #13697.
- mh := httpserver.Instrument(reg, logger, httpserver.AddRequestIDs(httpserver.LogRequests(h)))
- h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler())
- srv.Handler = mh
- srv.BaseContext = func(net.Listener) context.Context { return ctx }
- var listen arvados.URL
- for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs {
- break
- }
- if len(srv.Config.cluster.Services.WebDAV.InternalURLs) > 1 {
- logrus.Warn("Services.WebDAV.InternalURLs has more than one key; picked: ", listen)
- }
- srv.Addr = listen.Host
- return srv.Server.Start()
-}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
"io/ioutil"
"net"
"net/http"
+ "net/http/httptest"
"os"
"os/exec"
+ "regexp"
"strings"
"testing"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
// IntegrationSuite tests need an API server and a keep-web server
type IntegrationSuite struct {
- testServer *server
- ArvConfig *arvados.Config
+ testServer *httptest.Server
+ handler *handler
}
func (s *IntegrationSuite) TestNoToken(c *check.C) {
}
func (s *IntegrationSuite) Test200(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
for _, spec := range []curlCase{
// My collection
{
// Return header block and body.
func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
- testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
+ testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
if strings.Contains(auth, " ") {
// caller supplied entire Authorization header value
return
}
+// Run a full-featured server, including the metrics/health routes
+// that are added by service.Command.
+func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
+ logbuf = &bytes.Buffer{}
+ cluster = *s.handler.Cluster
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+
+ var configjson bytes.Buffer
+ json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
+ go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
+ for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
+ if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
+ srvaddr = "http://" + m[1]
+ break
+ }
+ }
+ if srvaddr == "" {
+ c.Fatal("timed out")
+ }
+ return
+}
+
+// Ensure uploads can take longer than API.RequestTimeout.
+//
+// Currently, this works only by accident: service.Command cancels the
+// request context as usual (there is no exemption), but
+// webdav.Handler doesn't notice if the request context is cancelled
+// while waiting to send or receive file data.
+func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
+ s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
+ _, srvaddr, _ := s.runServer(c)
+
+ var coll arvados.Collection
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveTokenV2
+ err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+ c.Assert(err, check.IsNil)
+
+ pr, pw := io.Pipe()
+ go func() {
+ time.Sleep(time.Second)
+ pw.Write(make([]byte, 10000000))
+ pw.Close()
+ }()
+ req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ time.Sleep(time.Second)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(len(body), check.Equals, 10000000)
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+ cluster, srvaddr, _ := s.runServer(c)
+ req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
+ req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ body, _ := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
+}
+
func (s *IntegrationSuite) TestMetrics(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
- origin := "http://" + s.testServer.Addr
- req, _ := http.NewRequest("GET", origin+"/notfound", nil)
+ cluster, srvaddr, _ := s.runServer(c)
+
+ req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
_, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
for i := 0; i < 2; i++ {
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
req.Host = arvadostest.FooCollection + ".example.com"
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err = http.DefaultClient.Do(req)
resp.Body.Close()
}
- s.testServer.Config.Cache.updateGauges()
+ time.Sleep(metricsUpdateInterval * 2)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer badtoken")
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
- // must not intercept that route.
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
- req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp, err = http.DefaultClient.Do(req)
- c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ // must not intercept that route. Ditto health check paths.
+ for _, path := range []string{"/metrics.json", "/_health/ping"} {
+ c.Logf("path: %q", path)
+ req, _ = http.NewRequest("GET", srvaddr+path, nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ }
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
- ldr.Path = "-"
- arvCfg, err := ldr.Load()
- c.Check(err, check.IsNil)
- cfg := newConfig(ctxlog.TestLogger(c), arvCfg)
- c.Assert(err, check.IsNil)
- cfg.Client = arvados.Client{
- APIHost: testAPIHost,
- Insecure: true,
- }
- listen := "127.0.0.1:0"
- cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.ManagementToken = arvadostest.ManagementToken
- cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- s.ArvConfig = arvCfg
- s.testServer = &server{Config: cfg}
logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(&bytes.Buffer{}, logger)
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+
ctx := ctxlog.Context(context.Background(), logger)
- err = s.testServer.Start(ctx, logger)
- c.Assert(err, check.Equals, nil)
+
+ s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
+ s.testServer = httptest.NewUnstartedServer(
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ s.handler)))
+ s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
+ s.testServer.Start()
+
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
- var err error
if s.testServer != nil {
- err = s.testServer.Close()
+ s.testServer.Close()
}
- c.Check(err, check.Equals, nil)
}
// Gocheck boilerplate
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"encoding/json"
"net/url"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
"gopkg.in/check.v1"
)
func (s *UnitSuite) TestStatus(c *check.C) {
- h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
u, _ := url.Parse("http://keep-web.example/status.json")
req := &http.Request{
Method: "GET",
RequestURI: u.RequestURI(),
}
resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
var status map[string]interface{}
},
}
resp := httptest.NewRecorder()
- s.testServer.Handler.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"crypto/rand"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import "golang.org/x/net/webdav"
FROM arvados/arvbox-base
ARG arvados_version
-ARG composer_version=arvados-fork
ARG workbench2_version=main
RUN cd /usr/src && \
git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
- git clone --no-checkout https://github.com/arvados/composer.git && \
- git -C composer checkout ${composer_version} && \
git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
git -C workbench2 checkout ${workbench2_version} && \
chown -R 1000:1000 /usr/src
RUN /usr/local/lib/arvbox/createusers.sh
RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/composer/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/workbench2/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/keep-web/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/workbench/run-service --only-deps
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
ExternalURL: "https://$localip:${services[keep-web-dl-ssl]}/"
- Composer:
- ExternalURL: "https://$localip:${services[composer]}"
Controller:
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
BlobSigningKey: $blob_signing_key
DefaultReplication: 1
TrustAllContent: true
+ Containers:
+ RuntimeEngine: singularity
Login:
Test:
Enable: true
[api]=8004
[controller]=8003
[controller-ssl]=8000
- [composer]=4200
[arv-git-httpd-ssl]=9000
[arv-git-httpd]=9001
[keep-web]=9003
. /usr/local/lib/arvbox/common.sh
. /usr/local/lib/arvbox/go-setup.sh
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/arv-git-httpd"
-install $GOPATH/bin/arv-git-httpd /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server arvados-git-httpd)
if test "$1" = "--only-deps" ; then
exit
fi
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
+flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
export PATH="$PATH:$ARVADOS_CONTAINER_PATH/git/bin"
cd ~git
-
-exec /usr/local/bin/arv-git-httpd
+exec /usr/local/bin/arvados-git-httpd
+++ /dev/null
-/usr/local/lib/arvbox/logger
\ No newline at end of file
+++ /dev/null
-#!/bin/sh
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -e
-
-exec /usr/local/lib/arvbox/runsu.sh $0-service $1
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-
-cd /usr/src/composer
-
-npm -d install --prefix /usr/local --global yarn@1.17.3
-
-yarn install
-
-if test "$1" = "--only-deps" ; then
- exit
-fi
-
-cat >/usr/src/composer/src/composer.yml <<EOF
-API_HOST: ${localip}:${services[controller-ssl]}
-EOF
-exec node_modules/.bin/ng serve --host 0.0.0.0 --port 4200 --env=webdev --ssl=true --sslCert="${server_cert}" --sslKey="${server_cert_key}"
. /usr/local/lib/arvbox/common.sh
. /usr/local/lib/arvbox/go-setup.sh
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/keep-web"
-install $GOPATH/bin/keep-web /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server keep-web)
if test "$1" = "--only-deps" ; then
exit
fi
+flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
exec /usr/local/bin/keep-web
exit
fi
-/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
-exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/keepproxy
+exec /usr/local/bin/keepproxy
cd /usr/src/workbench2
-npm -d install --prefix /usr/local --global yarn@1.17.3
-
yarn install
if test "$1" = "--only-deps" ; then