# initialize git_internal_dir
# usually /var/lib/arvados/internal.git (set in application.default.yml )
if [ "$APPLICATION_READY" = "1" ]; then
- GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:check 2>&1 | grep git_internal_dir | awk '{ print $2 }')
+ GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:dump 2>&1 | grep GitInternalDir | awk '{ print $2 }' |tr -d '"')
if [ ! -e "$GIT_INTERNAL_DIR" ]; then
run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
mkdir -p "$GIT_INTERNAL_DIR"
echo >&2
echo >&2 "$0 options:"
echo >&2 " -t, --tags version tag for docker"
- echo >&2 " -r, --repo Arvados package repot to use: dev, testing, stable (default: dev)"
+ echo >&2 " -r, --repo Arvados package repo to use: dev (default), testing, stable"
echo >&2 " -u, --upload Upload the images (docker push)"
echo >&2 " --no-cache Don't use build cache"
echo >&2 " -h, --help Display this help and exit"
LICENSE_PACKAGE_TS=20151208015500
if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
- RAILS_PACKAGE_ITERATION=8
+ RAILS_PACKAGE_ITERATION=1
else
RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
fi
echo "${svc} pid ${pid} ok"
}
+checkhealth() {
+ svc="$1"
+ port="$(cat "$WORKSPACE/tmp/${svc}.port")"
+ scheme=http
+ if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
+ scheme=https
+ fi
+ url="$scheme://localhost:${port}/_health/ping"
+ if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
+ echo "${url} failed"
+ return 1
+ fi
+}
+
checkdiscoverydoc() {
dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
&& checkdiscoverydoc $ARVADOS_API_HOST \
&& python sdk/python/tests/run_test_server.py start_controller \
&& checkpidfile controller \
+ && checkhealth controller \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& checkpidfile keepproxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& checkpidfile keep-web \
+ && checkhealth keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& checkpidfile arv-git-httpd \
+ && checkhealth arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
&& checkpidfile ws \
&& eval $(python sdk/python/tests/run_test_server.py start_nginx) \
lib/cloud
lib/cloud/azure
lib/cloud/ec2
+ lib/config
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
test_services/api() {
rm -f "$WORKSPACE/services/api/git-commit.version"
cd "$WORKSPACE/services/api" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec bin/rails test TESTOPTS='-v -d' ${testargs[services/api]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]}
}
test_sdk/ruby() {
"os"
"git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/lib/controller"
"git.curoverse.com/arvados.git/lib/dispatchcloud"
)
"-version": cmd.Version(version),
"--version": cmd.Version(version),
+ "config-check": config.CheckCommand,
+ "config-dump": config.DumpCommand,
"controller": controller.Command,
"dispatch-cloud": dispatchcloud.Command,
})
require "rubygems"
require "colorize"
-#task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
-task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
+task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
vars = ['baseurl', 'arvados_cluster_uuid', 'arvados_api_host', 'arvados_workbench_host']
vars.each do |v|
if ENV[v]
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/ghodss/yaml"
+)
+
+var DumpCommand cmd.Handler = dumpCommand{}
+
+type dumpCommand struct{}
+
+func (dumpCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ defer func() {
+ if err != nil {
+ fmt.Fprintf(stderr, "%s\n", err)
+ }
+ }()
+ if len(args) != 0 {
+ err = fmt.Errorf("usage: %s <config-src.yaml >config-min.yaml", prog)
+ return 2
+ }
+ log := ctxlog.New(stderr, "text", "info")
+ cfg, err := Load(stdin, log)
+ if err != nil {
+ return 1
+ }
+ out, err := yaml.Marshal(cfg)
+ if err != nil {
+ return 1
+ }
+ _, err = stdout.Write(out)
+ if err != nil {
+ return 1
+ }
+ return 0
+}
+
+var CheckCommand cmd.Handler = checkCommand{}
+
+type checkCommand struct{}
+
+func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ defer func() {
+ if err != nil {
+ fmt.Fprintf(stderr, "%s\n", err)
+ }
+ }()
+ if len(args) != 0 {
+ err = fmt.Errorf("usage: %s <config-src.yaml && echo 'no changes needed'", prog)
+ return 2
+ }
+ log := &plainLogger{w: stderr}
+ buf, err := ioutil.ReadAll(stdin)
+ if err != nil {
+ return 1
+ }
+ withoutDepr, err := load(bytes.NewBuffer(buf), log, false)
+ if err != nil {
+ return 1
+ }
+ withDepr, err := load(bytes.NewBuffer(buf), nil, true)
+ if err != nil {
+ return 1
+ }
+ cmd := exec.Command("diff", "-u", "--label", "without-deprecated-configs", "--label", "relying-on-deprecated-configs", "/dev/fd/3", "/dev/fd/4")
+ for _, obj := range []interface{}{withoutDepr, withDepr} {
+ y, _ := yaml.Marshal(obj)
+ pr, pw, err := os.Pipe()
+ if err != nil {
+ return 1
+ }
+ defer pr.Close()
+ go func() {
+ io.Copy(pw, bytes.NewBuffer(y))
+ pw.Close()
+ }()
+ cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
+ }
+ diff, err := cmd.CombinedOutput()
+ if bytes.HasPrefix(diff, []byte("--- ")) {
+ fmt.Fprintln(stdout, "Your configuration is relying on deprecated entries. Suggest making the following changes.")
+ stdout.Write(diff)
+ return 1
+ } else if len(diff) > 0 {
+ fmt.Fprintf(stderr, "Unexpected diff output:\n%s", diff)
+ return 1
+ } else if err != nil {
+ return 1
+ }
+ if log.used {
+ return 1
+ }
+ return 0
+}
+
+type plainLogger struct {
+ w io.Writer
+ used bool
+}
+
+func (pl *plainLogger) Warnf(format string, args ...interface{}) {
+ pl.used = true
+ fmt.Fprintf(pl.w, format+"\n", args...)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (s *CommandSuite) TestBadArg(c *check.C) {
+ var stderr bytes.Buffer
+ code := DumpCommand.RunCommand("arvados config-dump", []string{"-badarg"}, bytes.NewBuffer(nil), bytes.NewBuffer(nil), &stderr)
+ c.Check(code, check.Equals, 2)
+ c.Check(stderr.String(), check.Matches, `(?ms)usage: .*`)
+}
+
+func (s *CommandSuite) TestEmptyInput(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ code := DumpCommand.RunCommand("arvados config-dump", nil, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(code, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `config does not define any clusters\n`)
+}
+
+func (s *CommandSuite) TestCheckNoDeprecatedKeys(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ API:
+ MaxItemsPerResponse: 1234
+`
+ code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Check(code, check.Equals, 0)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Equals, "")
+}
+
+func (s *CommandSuite) TestCheckDeprecatedKeys(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ RequestLimits:
+ MaxItemsPerResponse: 1234
+`
+ code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Check(code, check.Equals, 1)
+ c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
+}
+
+func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ Bogus1: foo
+ BogusSection:
+ Bogus2: foo
+ API:
+ Bogus3:
+ Bogus4: true
+ PostgreSQL:
+ ConnectionPool:
+ {Bogus5: true}
+`
+ code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Log(stderr.String())
+ c.Check(code, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.Bogus1\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.BogusSection\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.API.Bogus3\n.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*unexpected object in config entry: Clusters.z1234.PostgreSQL.ConnectionPool\n.*`)
+}
+
+func (s *CommandSuite) TestDumpUnknownKey(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ UnknownKey: foobar
+ ManagementToken: secret
+`
+ code := DumpCommand.RunCommand("arvados config-dump", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Check(code, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.UnknownKey.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n z1234:\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
+ c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
+}
# All parameters here are passed to the PG client library in a connection string;
# see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
Host: ""
- Port: 0
+ Port: ""
User: ""
Password: ""
DBName: ""
# update on the permission view in the future, if not already scheduled.
AsyncPermissionsUpdateInterval: 20
+ # Maximum number of concurrent outgoing requests to make while
+ # serving a single incoming multi-cluster (federated) request.
+ MaxRequestAmplification: 4
+
# RailsSessionSecretToken is a string of alphanumeric characters
# used by Rails to sign session tokens. IMPORTANT: This is a
# site secret. It should be at least 50 characters.
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/ghodss/yaml"
+)
+
+type deprRequestLimits struct {
+ MaxItemsPerResponse *int
+ MultiClusterRequestConcurrency *int
+}
+
+type deprCluster struct {
+ RequestLimits deprRequestLimits
+ NodeProfiles map[string]arvados.NodeProfile
+}
+
+type deprecatedConfig struct {
+ Clusters map[string]deprCluster
+}
+
+func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) error {
+ var dc deprecatedConfig
+ err := yaml.Unmarshal(configdata, &dc)
+ if err != nil {
+ return err
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ return err
+ }
+ for id, dcluster := range dc.Clusters {
+ cluster, ok := cfg.Clusters[id]
+ if !ok {
+ return fmt.Errorf("can't load legacy config %q that is not present in current config", id)
+ }
+ for name, np := range dcluster.NodeProfiles {
+ if name == "*" || name == os.Getenv("ARVADOS_NODE_PROFILE") || name == hostname {
+ name = "localhost"
+ } else if log != nil {
+ log.Warnf("overriding Clusters.%s.Services using Clusters.%s.NodeProfiles.%s (guessing %q is a hostname)", id, id, name, name)
+ }
+ applyDeprecatedNodeProfile(name, np.RailsAPI, &cluster.Services.RailsAPI)
+ applyDeprecatedNodeProfile(name, np.Controller, &cluster.Services.Controller)
+ applyDeprecatedNodeProfile(name, np.DispatchCloud, &cluster.Services.DispatchCloud)
+ }
+ if dst, n := &cluster.API.MaxItemsPerResponse, dcluster.RequestLimits.MaxItemsPerResponse; n != nil && *n != *dst {
+ *dst = *n
+ }
+ if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
+ *dst = *n
+ }
+ cfg.Clusters[id] = cluster
+ }
+ return nil
+}
+
+func applyDeprecatedNodeProfile(hostname string, ssi arvados.SystemServiceInstance, svc *arvados.Service) {
+ scheme := "https"
+ if !ssi.TLS {
+ scheme = "http"
+ }
+ if svc.InternalURLs == nil {
+ svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
+ }
+ host := ssi.Listen
+ if host == "" {
+ return
+ }
+ if strings.HasPrefix(host, ":") {
+ host = hostname + host
+ }
+ svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ s.checkEquivalent(c, `
+Clusters:
+ z1111:
+ NodeProfiles:
+ "*":
+ arvados-controller:
+ listen: ":9004"
+ `+hostname+`:
+ arvados-api-server:
+ listen: ":8000"
+ dispatch-host:
+ arvados-dispatch-cloud:
+ listen: ":9006"
+`, `
+Clusters:
+ z1111:
+ Services:
+ RailsAPI:
+ InternalURLs:
+ "http://localhost:8000": {}
+ Controller:
+ InternalURLs:
+ "http://localhost:9004": {}
+ DispatchCloud:
+ InternalURLs:
+ "http://dispatch-host:9006": {}
+ NodeProfiles:
+ "*":
+ arvados-controller:
+ listen: ":9004"
+ `+hostname+`:
+ arvados-api-server:
+ listen: ":8000"
+ dispatch-host:
+ arvados-dispatch-cloud:
+ listen: ":9006"
+`)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// +build ignore
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+)
+
+func main() {
+ err := generate()
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func generate() error {
+ outfn := "generated_config.go"
+ tmpfile, err := ioutil.TempFile(".", "."+outfn+".")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmpfile.Name())
+
+ gofmt := exec.Command("gofmt", "-s")
+ gofmt.Stdout = tmpfile
+ gofmt.Stderr = os.Stderr
+ w, err := gofmt.StdinPipe()
+ if err != nil {
+ return err
+ }
+ gofmt.Start()
+
+ // copyright header: same as this file
+ cmd := exec.Command("head", "-n", "4", "generate.go")
+ cmd.Stdout = w
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ if err != nil {
+ return err
+ }
+
+ data, err := ioutil.ReadFile("config.default.yml")
+ if err != nil {
+ return err
+ }
+ _, err = fmt.Fprintf(w, "package config\nvar DefaultYAML = []byte(`%s`)", bytes.Replace(data, []byte{'`'}, []byte("`+\"`\"+`"), -1))
+ if err != nil {
+ return err
+ }
+ err = w.Close()
+ if err != nil {
+ return err
+ }
+ err = gofmt.Wait()
+ if err != nil {
+ return err
+ }
+ err = tmpfile.Close()
+ if err != nil {
+ return err
+ }
+ return os.Rename(tmpfile.Name(), outfn)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+var DefaultYAML = []byte(`# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Do not use this file for site configuration. Create
+# /etc/arvados/config.yml instead.
+#
+# The order of precedence (highest to lowest):
+# 1. Legacy component-specific config files (deprecated)
+# 2. /etc/arvados/config.yml
+# 3. config.default.yml
+
+Clusters:
+ xxxxx:
+ SystemRootToken: ""
+
+ # Token to be included in all healthcheck requests. Disabled by default.
+ # Server expects request header of the format "Authorization: Bearer xxx"
+ ManagementToken: ""
+
+ Services:
+ RailsAPI:
+ InternalURLs: {}
+ GitHTTP:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepstore:
+ InternalURLs: {}
+ Controller:
+ InternalURLs: {}
+ ExternalURL: ""
+ Websocket:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepbalance:
+ InternalURLs: {}
+ GitHTTP:
+ InternalURLs: {}
+ ExternalURL: ""
+ GitSSH:
+ ExternalURL: ""
+ DispatchCloud:
+ InternalURLs: {}
+ SSO:
+ ExternalURL: ""
+ Keepproxy:
+ InternalURLs: {}
+ ExternalURL: ""
+ WebDAV:
+ InternalURLs: {}
+ ExternalURL: ""
+ WebDAVDownload:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepstore:
+ InternalURLs: {}
+ Composer:
+ ExternalURL: ""
+ WebShell:
+ ExternalURL: ""
+ Workbench1:
+ InternalURLs: {}
+ ExternalURL: ""
+ Workbench2:
+ ExternalURL: ""
+ PostgreSQL:
+ # max concurrent connections per arvados server daemon
+ ConnectionPool: 32
+ Connection:
+ # All parameters here are passed to the PG client library in a connection string;
+ # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+ Host: ""
+ Port: ""
+ User: ""
+ Password: ""
+ DBName: ""
+ API:
+ # Maximum size (in bytes) allowed for a single API request. This
+ # limit is published in the discovery document for use by clients.
+ # Note: You must separately configure the upstream web server or
+ # proxy to actually enforce the desired maximum request size on the
+ # server side.
+ MaxRequestSize: 134217728
+
+ # Limit the number of bytes read from the database during an index
+ # request (by retrieving and returning fewer rows than would
+ # normally be returned in a single response).
+ # Note 1: This setting never reduces the number of returned rows to
+ # zero, no matter how big the first data row is.
+ # Note 2: Currently, this is only checked against a specific set of
+ # columns that tend to get large (collections.manifest_text,
+ # containers.mounts, workflows.definition). Other fields (e.g.,
+ # "properties" hashes) are not counted against this limit.
+ MaxIndexDatabaseRead: 134217728
+
+ # Maximum number of items to return when responding to a APIs that
+ # can return partial result sets using limit and offset parameters
+ # (e.g., *.index, groups.contents). If a request specifies a "limit"
+ # parameter higher than this value, this value is used instead.
+ MaxItemsPerResponse: 1000
+
+ # API methods to disable. Disabled methods are not listed in the
+ # discovery document, and respond 404 to all requests.
+ # Example: ["jobs.create", "pipeline_instances.create"]
+ DisabledAPIs: []
+
+ # Interval (seconds) between asynchronous permission view updates. Any
+ # permission-updating API called with the 'async' parameter schedules a an
+ # update on the permission view in the future, if not already scheduled.
+ AsyncPermissionsUpdateInterval: 20
+
+ # Maximum number of concurrent outgoing requests to make while
+ # serving a single incoming multi-cluster (federated) request.
+ MaxRequestAmplification: 4
+
+ # RailsSessionSecretToken is a string of alphanumeric characters
+ # used by Rails to sign session tokens. IMPORTANT: This is a
+ # site secret. It should be at least 50 characters.
+ RailsSessionSecretToken: ""
+
+ Users:
+ # Config parameters to automatically setup new users. If enabled,
+ # this users will be able to self-activate. Enable this if you want
+ # to run an open instance where anyone can create an account and use
+ # the system without requiring manual approval.
+ #
+ # The params auto_setup_new_users_with_* are meaningful only when auto_setup_new_users is turned on.
+ # auto_setup_name_blacklist is a list of usernames to be blacklisted for auto setup.
+ AutoSetupNewUsers: false
+ AutoSetupNewUsersWithVmUUID: ""
+ AutoSetupNewUsersWithRepository: false
+ AutoSetupUsernameBlacklist: [arvados, git, gitolite, gitolite-admin, root, syslog]
+
+ # When new_users_are_active is set to true, new users will be active
+ # immediately. This skips the "self-activate" step which enforces
+ # user agreements. Should only be enabled for development.
+ NewUsersAreActive: false
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses OAuth2 against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ AutoAdminUserWithEmail: ""
+
+ # If auto_admin_first_user is set to true, the first user to log in when no
+ # other admin users exist will automatically become an admin user.
+ AutoAdminFirstUser: false
+
+ # Email address to notify whenever a user creates a profile for the
+ # first time
+ UserProfileNotificationAddress: ""
+ AdminNotifierEmailFrom: arvados@example.com
+ EmailSubjectPrefix: "[ARVADOS] "
+ UserNotifierEmailFrom: arvados@example.com
+ NewUserNotificationRecipients: []
+ NewInactiveUserNotificationRecipients: []
+
+ AuditLogs:
+ # Time to keep audit logs, in seconds. (An audit log is a row added
+ # to the "logs" table in the PostgreSQL database each time an
+ # Arvados object is created, modified, or deleted.)
+ #
+ # Currently, websocket event notifications rely on audit logs, so
+ # this should not be set lower than 600 (5 minutes).
+ MaxAge: 1209600
+
+ # Maximum number of log rows to delete in a single SQL transaction.
+ #
+ # If max_audit_log_delete_batch is 0, log entries will never be
+ # deleted by Arvados. Cleanup can be done by an external process
+ # without affecting any Arvados system processes, as long as very
+ # recent (<5 minutes old) logs are not deleted.
+ #
+ # 100000 is a reasonable batch size for most sites.
+ MaxDeleteBatch: 0
+
+ # Attributes to suppress in events and audit logs. Notably,
+ # specifying ["manifest_text"] here typically makes the database
+ # smaller and faster.
+ #
+ # Warning: Using any non-empty value here can have undesirable side
+ # effects for any client or component that relies on event logs.
+ # Use at your own risk.
+ UnloggedAttributes: []
+
+ SystemLogs:
+ # Maximum characters of (JSON-encoded) query parameters to include
+ # in each request log entry. When params exceed this size, they will
+ # be JSON-encoded, truncated to this size, and logged as
+ # params_truncated.
+ MaxRequestLogParamsSize: 2000
+
+ Collections:
+ # Allow clients to create collections by providing a manifest with
+ # unsigned data blob locators. IMPORTANT: This effectively disables
+ # access controls for data stored in Keep: a client who knows a hash
+ # can write a manifest that references the hash, pass it to
+ # collections.create (which will create a permission link), use
+ # collections.get to obtain a signature for that data locator, and
+ # use that signed locator to retrieve the data from Keep. Therefore,
+ # do not turn this on if your users expect to keep data private from
+ # one another!
+ BlobSigning: true
+
+ # blob_signing_key is a string of alphanumeric characters used to
+ # generate permission signatures for Keep locators. It must be
+ # identical to the permission key given to Keep. IMPORTANT: This is
+ # a site secret. It should be at least 50 characters.
+ #
+ # Modifying blob_signing_key will invalidate all existing
+ # signatures, which can cause programs to fail (e.g., arv-put,
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
+ # no such processes are running.
+ BlobSigningKey: ""
+
+ # Default replication level for collections. This is used when a
+ # collection's replication_desired attribute is nil.
+ DefaultReplication: 2
+
+ # Lifetime (in seconds) of blob permission signatures generated by
+ # the API server. This determines how long a client can take (after
+ # retrieving a collection record) to retrieve the collection data
+ # from Keep. If the client needs more time than that (assuming the
+ # collection still has the same content and the relevant user/token
+ # still has permission) the client can retrieve the collection again
+ # to get fresh signatures.
+ #
+ # This must be exactly equal to the -blob-signature-ttl flag used by
+ # keepstore servers. Otherwise, reading data blocks and saving
+ # collections will fail with HTTP 403 permission errors.
+ #
+ # Modifying blob_signature_ttl invalidates existing signatures; see
+ # blob_signing_key note above.
+ #
+ # The default is 2 weeks.
+ BlobSigningTTL: 1209600
+
+ # Default lifetime for ephemeral collections: 2 weeks. This must not
+ # be less than blob_signature_ttl.
+ DefaultTrashLifetime: 1209600
+
+ # Interval (seconds) between trash sweeps. During a trash sweep,
+ # collections are marked as trash if their trash_at time has
+ # arrived, and deleted if their delete_at time has arrived.
+ TrashSweepInterval: 60
+
+ # If true, enable collection versioning.
+ # When a collection's preserve_version field is true or the current version
+ # is older than the amount of seconds defined on preserve_version_if_idle,
+ # a snapshot of the collection's previous state is created and linked to
+ # the current collection.
+ CollectionVersioning: false
+
+ # 0 = auto-create a new version on every update.
+ # -1 = never auto-create new versions.
+ # > 0 = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1
+
+ Login:
+ # These settings are provided by your OAuth2 provider (e.g.,
+ # sso-provider).
+ ProviderAppSecret: ""
+ ProviderAppID: ""
+
+ Git:
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ Repositories: /var/lib/arvados/git/repositories
+
+ TLS:
+ Insecure: false
+
+ Containers:
+ # List of supported Docker Registry image formats that compute nodes
+ # are able to use. ` + "`" + `arv keep docker` + "`" + ` will error out if a user tries
+ # to store an image with an unsupported format. Use an empty array
+ # to skip the compatibility check (and display a warning message to
+ # that effect).
+ #
+ # Example for sites running docker < 1.10: ["v1"]
+ # Example for sites running docker >= 1.10: ["v2"]
+ # Example for disabling check: []
+ SupportedDockerImageFormats: ["v2"]
+
+ # Include details about job reuse decisions in the server log. This
+ # causes additional database queries to run, so it should not be
+ # enabled unless you expect to examine the resulting logs for
+ # troubleshooting purposes.
+ LogReuseDecisions: false
+
+ # Default value for keep_cache_ram of a container's runtime_constraints.
+ DefaultKeepCacheRAM: 268435456
+
+ # Number of times a container can be unlocked before being
+ # automatically cancelled.
+ MaxDispatchAttempts: 5
+
+ # Default value for container_count_max for container requests. This is the
+ # number of times Arvados will create a new container to satisfy a container
+ # request. If a container is cancelled it will retry a new container if
+ # container_count < container_count_max on any container requests associated
+ # with the cancelled container.
+ MaxRetryAttempts: 3
+
+ # The maximum number of compute nodes that can be in use simultaneously
+ # If this limit is reduced, any existing nodes with slot number >= new limit
+ # will not be counted against the new limit. In other words, the new limit
+ # won't be strictly enforced until those nodes with higher slot numbers
+ # go down.
+ MaxComputeVMs: 64
+
+ # Preemptible instance support (e.g. AWS Spot Instances)
+ # When true, child containers will get created with the preemptible
+ # scheduling parameter parameter set.
+ UsePreemptibleInstances: false
+
+ # Include details about job reuse decisions in the server log. This
+ # causes additional database queries to run, so it should not be
+ # enabled unless you expect to examine the resulting logs for
+ # troubleshooting purposes.
+ LogReuseDecisions: false
+
+ Logging:
+ # When you run the db:delete_old_container_logs task, it will find
+ # containers that have been finished for at least this many seconds,
+ # and delete their stdout, stderr, arv-mount, crunch-run, and
+ # crunchstat logs from the logs table.
+ MaxAge: 720h
+
+ # These two settings control how frequently log events are flushed to the
+ # database. Log lines are buffered until either crunch_log_bytes_per_event
+ # has been reached or crunch_log_seconds_between_events has elapsed since
+ # the last flush.
+ LogBytesPerEvent: 4096
+ LogSecondsBetweenEvents: 1
+
+ # The sample period for throttling logs, in seconds.
+ LogThrottlePeriod: 60
+
+ # Maximum number of bytes that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ LogThrottleBytes: 65536
+
+ # Maximum number of lines that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ LogThrottleLines: 1024
+
+ # Maximum bytes that may be logged by a single job. Log bytes that are
+ # silenced by throttling are not counted against this total.
+ LimitLogBytesPerJob: 67108864
+
+ LogPartialLineThrottlePeriod: 5
+
+ # Container logs are written to Keep and saved in a collection,
+ # which is updated periodically while the container runs. This
+ # value sets the interval (given in seconds) between collection
+ # updates.
+ LogUpdatePeriod: 1800
+
+ # The log collection is also updated when the specified amount of
+ # log data (given in bytes) is produced in less than one update
+ # period.
+ LogUpdateSize: 33554432
+
+ SLURM:
+ Managed:
+ # Path to dns server configuration directory
+ # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
+ # files or touch restart.txt (see below).
+ DNSServerConfDir: ""
+
+ # Template file for the dns server host snippets. See
+ # unbound.template in this directory for an example. If false, do
+ # not write any config files.
+ DNSServerConfTemplate: ""
+
+ # String to write to {dns_server_conf_dir}/restart.txt (with a
+ # trailing newline) after updating local data. If false, do not
+ # open or write the restart.txt file.
+ DNSServerReloadCommand: ""
+
+ # Command to run after each DNS update. Template variables will be
+ # substituted; see the "unbound" example below. If false, do not run
+ # a command.
+ DNSServerUpdateCommand: ""
+
+ ComputeNodeDomain: ""
+ ComputeNodeNameservers:
+ - 192.168.1.1
+
+ # Hostname to assign to a compute node when it sends a "ping" and the
+ # hostname in its Node record is nil.
+ # During bootstrapping, the "ping" script is expected to notice the
+ # hostname given in the ping response, and update its unix hostname
+ # accordingly.
+ # If false, leave the hostname alone (this is appropriate if your compute
+ # nodes' hostnames are already assigned by some other mechanism).
+ #
+ # One way or another, the hostnames of your node records should agree
+ # with your DNS records and your /etc/slurm-llnl/slurm.conf files.
+ #
+ # Example for compute0000, compute0001, ....:
+ # assign_node_hostname: compute%<slot_number>04d
+ # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
+ AssignNodeHostname: "compute%<slot_number>d"
+
+ JobsAPI:
+ # Enable the legacy Jobs API. This value must be a string.
+ # 'auto' -- (default) enable the Jobs API only if it has been used before
+ # (i.e., there are job records in the database)
+ # 'true' -- enable the Jobs API despite lack of existing records.
+ # 'false' -- disable the Jobs API despite presence of existing records.
+ Enable: 'auto'
+
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ GitInternalDir: /var/lib/arvados/internal.git
+
+ # Docker image to be used when none found in runtime_constraints of a job
+ DefaultDockerImage: ""
+
+ # none or slurm_immediate
+ CrunchJobWrapper: none
+
+ # username, or false = do not set uid when running jobs.
+ CrunchJobUser: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ CrunchRefreshTrigger: /tmp/crunch_refresh_trigger
+
+ # Control job reuse behavior when two completed jobs match the
+ # search criteria and have different outputs.
+ #
+ # If true, in case of a conflict, reuse the earliest job (this is
+ # similar to container reuse behavior).
+ #
+ # If false, in case of a conflict, do not reuse any completed job,
+ # but do reuse an already-running job if available (this is the
+ # original job reuse behavior, and is still the default).
+ ReuseJobIfOutputsDiffer: false
+
+ Mail:
+ MailchimpAPIKey: ""
+ MailchimpListID: ""
+ SendUserSetupNotificationEmail: ""
+ IssueReporterEmailFrom: ""
+ IssueReporterEmailTo: ""
+ SupportEmailAddress: ""
+ EmailFrom: ""
+ RemoteClusters:
+ "*":
+ Proxy: false
+ ActivateUsers: false
+`)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/ghodss/yaml"
+ "github.com/imdario/mergo"
+)
+
+type logger interface {
+ Warnf(string, ...interface{})
+}
+
+func LoadFile(path string, log logger) (*arvados.Config, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return Load(f, log)
+}
+
+func Load(rdr io.Reader, log logger) (*arvados.Config, error) {
+ return load(rdr, log, true)
+}
+
+func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error) {
+ buf, err := ioutil.ReadAll(rdr)
+ if err != nil {
+ return nil, err
+ }
+
+ // Load the config into a dummy map to get the cluster ID
+ // keys, discarding the values; then set up defaults for each
+ // cluster ID; then load the real config on top of the
+ // defaults.
+ var dummy struct {
+ Clusters map[string]struct{}
+ }
+ err = yaml.Unmarshal(buf, &dummy)
+ if err != nil {
+ return nil, err
+ }
+ if len(dummy.Clusters) == 0 {
+ return nil, errors.New("config does not define any clusters")
+ }
+
+ // We can't merge deep structs here; instead, we unmarshal the
+ // default & loaded config files into generic maps, merge
+ // those, and then json-encode+decode the result into the
+ // config struct type.
+ var merged map[string]interface{}
+ for id := range dummy.Clusters {
+ var src map[string]interface{}
+ err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte(" xxxxx:"), []byte(" "+id+":"), -1), &src)
+ if err != nil {
+ return nil, fmt.Errorf("loading defaults for %s: %s", id, err)
+ }
+ err = mergo.Merge(&merged, src, mergo.WithOverride)
+ if err != nil {
+ return nil, fmt.Errorf("merging defaults for %s: %s", id, err)
+ }
+ }
+ var src map[string]interface{}
+ err = yaml.Unmarshal(buf, &src)
+ if err != nil {
+ return nil, fmt.Errorf("loading config data: %s", err)
+ }
+ logExtraKeys(log, merged, src, "")
+ err = mergo.Merge(&merged, src, mergo.WithOverride)
+ if err != nil {
+ return nil, fmt.Errorf("merging config data: %s", err)
+ }
+
+ // map[string]interface{} => json => arvados.Config
+ var cfg arvados.Config
+ var errEnc error
+ pr, pw := io.Pipe()
+ go func() {
+ errEnc = json.NewEncoder(pw).Encode(merged)
+ pw.Close()
+ }()
+ err = json.NewDecoder(pr).Decode(&cfg)
+ if errEnc != nil {
+ err = errEnc
+ }
+ if err != nil {
+ return nil, fmt.Errorf("transcoding config data: %s", err)
+ }
+
+ if useDeprecated {
+ err = applyDeprecatedConfig(&cfg, buf, log)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Check for known mistakes
+ for id, cc := range cfg.Clusters {
+ err = checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return &cfg, nil
+}
+
+func checkKeyConflict(label string, m map[string]string) error {
+ saw := map[string]bool{}
+ for k := range m {
+ k = strings.ToLower(k)
+ if saw[k] {
+ return fmt.Errorf("%s: multiple entries for %q (fix by using same capitalization as default/example file)", label, k)
+ }
+ saw[k] = true
+ }
+ return nil
+}
+
+func logExtraKeys(log logger, expected, supplied map[string]interface{}, prefix string) {
+ if log == nil {
+ return
+ }
+ for k, vsupp := range supplied {
+ if vexp, ok := expected[k]; !ok {
+ log.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
+ } else if vsupp, ok := vsupp.(map[string]interface{}); !ok {
+ // if vsupp is a map but vexp isn't map, this
+ // will be caught elsewhere; see TestBadType.
+ continue
+ } else if vexp, ok := vexp.(map[string]interface{}); !ok {
+ log.Warnf("unexpected object in config entry: %s%s", prefix, k)
+ } else {
+ logExtraKeys(log, vexp, vsupp, prefix+k+".")
+ }
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "os/exec"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/ghodss/yaml"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&LoadSuite{})
+
+type LoadSuite struct{}
+
+func (s *LoadSuite) TestEmpty(c *check.C) {
+ cfg, err := Load(&bytes.Buffer{}, ctxlog.TestLogger(c))
+ c.Check(cfg, check.IsNil)
+ c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
+}
+
+func (s *LoadSuite) TestNoConfigs(c *check.C) {
+ cfg, err := Load(bytes.NewBufferString(`Clusters: {"z1111": {}}`), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ c.Assert(cfg.Clusters, check.HasLen, 1)
+ cc, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(cc.ClusterID, check.Equals, "z1111")
+ c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
+ c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
+}
+
+func (s *LoadSuite) TestMultipleClusters(c *check.C) {
+ cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"z1111":{},"z2222":{}}}`), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ c1, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(c1.ClusterID, check.Equals, "z1111")
+ c2, err := cfg.GetCluster("z2222")
+ c.Assert(err, check.IsNil)
+ c.Check(c2.ClusterID, check.Equals, "z2222")
+}
+
+func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
+ _, err := Load(bytes.NewBufferString(`
+Clusters:
+ zzzzz:
+ postgresql:
+ connection:
+ dbname: dbname
+ host: host
+`), ctxlog.TestLogger(c))
+ c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
+}
+
+func (s *LoadSuite) TestBadType(c *check.C) {
+ for _, data := range []string{`
+Clusters:
+ zzzzz:
+ PostgreSQL: true
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: true
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: "foo"
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: []
+`, `
+Clusters:
+ zzzzz:
+ PostgreSQL:
+ ConnectionPool: [] # {foo: bar} isn't caught here; we rely on config-check
+`,
+ } {
+ c.Log(data)
+ v, err := Load(bytes.NewBufferString(data), ctxlog.TestLogger(c))
+ if v != nil {
+ c.Logf("%#v", v.Clusters["zzzzz"].PostgreSQL.ConnectionPool)
+ }
+ c.Check(err, check.ErrorMatches, `.*cannot unmarshal .*PostgreSQL.*`)
+ }
+}
+
+func (s *LoadSuite) TestMovedKeys(c *check.C) {
+ s.checkEquivalent(c, `# config has old keys only
+Clusters:
+ zzzzz:
+ RequestLimits:
+ MultiClusterRequestConcurrency: 3
+ MaxItemsPerResponse: 999
+`, `
+Clusters:
+ zzzzz:
+ API:
+ MaxRequestAmplification: 3
+ MaxItemsPerResponse: 999
+`)
+ s.checkEquivalent(c, `# config has both old and new keys; old values win
+Clusters:
+ zzzzz:
+ RequestLimits:
+ MultiClusterRequestConcurrency: 0
+ MaxItemsPerResponse: 555
+ API:
+ MaxRequestAmplification: 3
+ MaxItemsPerResponse: 999
+`, `
+Clusters:
+ zzzzz:
+ API:
+ MaxRequestAmplification: 0
+ MaxItemsPerResponse: 555
+`)
+}
+
+func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
+ got, err := Load(bytes.NewBufferString(goty), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ expected, err := Load(bytes.NewBufferString(expectedy), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ if !c.Check(got, check.DeepEquals, expected) {
+ cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
+ for _, obj := range []interface{}{expected, got} {
+ y, _ := yaml.Marshal(obj)
+ pr, pw, err := os.Pipe()
+ c.Assert(err, check.IsNil)
+ defer pr.Close()
+ go func() {
+ io.Copy(pw, bytes.NewBuffer(y))
+ pw.Close()
+ }()
+ cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
+ }
+ diff, err := cmd.CombinedOutput()
+ c.Log(string(diff))
+ c.Check(err, check.IsNil)
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+//go:generate go run generate.go
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "io/ioutil"
+ "testing"
+)
+
+func TestUpToDate(t *testing.T) {
+ src := "config.default.yml"
+ srcdata, err := ioutil.ReadFile(src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(srcdata, DefaultYAML) {
+ t.Fatalf("content of %s differs from DefaultYAML -- you need to run 'go generate' and commit", src)
+ }
+}
// returned to the client. When that happens, all
// other outstanding requests are cancelled
sharedContext, cancelFunc := context.WithCancel(req.Context())
+ defer cancelFunc()
+
req = req.WithContext(sharedContext)
wg := sync.WaitGroup{}
pdh := m[1]
success := make(chan *http.Response)
errorChan := make(chan error, len(h.handler.Cluster.RemoteClusters))
- // use channel as a semaphore to limit the number of concurrent
- // requests at a time
- sem := make(chan bool, h.handler.Cluster.RequestLimits.GetMultiClusterRequestConcurrency())
-
- defer cancelFunc()
+ acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
for remoteID := range h.handler.Cluster.RemoteClusters {
if remoteID == h.handler.Cluster.ClusterID {
wg.Add(1)
go func(remote string) {
defer wg.Done()
- // blocks until it can put a value into the
- // channel (which has a max queue capacity)
- sem <- true
+ acquire()
+ defer release()
select {
case <-sharedContext.Done():
return
case success <- newResponse:
wasSuccess = true
}
- <-sem
}(remoteID)
}
go func() {
httpserver.Error(w, "Federated multi-object may not provide 'limit', 'offset' or 'order'.", http.StatusBadRequest)
return true
}
- if expectCount > h.handler.Cluster.RequestLimits.GetMaxItemsPerResponse() {
+ if max := h.handler.Cluster.API.MaxItemsPerResponse; expectCount > max {
httpserver.Error(w, fmt.Sprintf("Federated multi-object request for %v objects which is more than max page size %v.",
- expectCount, h.handler.Cluster.RequestLimits.GetMaxItemsPerResponse()), http.StatusBadRequest)
+ expectCount, max), http.StatusBadRequest)
return true
}
if req.Form.Get("select") != "" {
// Perform concurrent requests to each cluster
- // use channel as a semaphore to limit the number of concurrent
- // requests at a time
- sem := make(chan bool, h.handler.Cluster.RequestLimits.GetMultiClusterRequestConcurrency())
- defer close(sem)
+ acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
wg := sync.WaitGroup{}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// Nothing to query
continue
}
-
- // blocks until it can put a value into the
- // channel (which has a max queue capacity)
- sem <- true
+ acquire()
wg.Add(1)
go func(k string, v []string) {
+ defer release()
+ defer wg.Done()
rp, kn, err := h.remoteQueryUUIDs(w, req, k, v)
mtx.Lock()
+ defer mtx.Unlock()
if err == nil {
completeResponses = append(completeResponses, rp...)
kind = kn
} else {
errors = append(errors, err)
}
- mtx.Unlock()
- wg.Done()
- <-sem
}(k, v)
}
wg.Wait()
NodeProfiles: map[string]arvados.NodeProfile{
"*": nodeProfile,
},
- RequestLimits: arvados.RequestLimits{
- MaxItemsPerResponse: 1000,
- MultiClusterRequestConcurrency: 4,
+ API: arvados.API{
+ MaxItemsPerResponse: 1000,
+ MaxRequestAmplification: 4,
},
}, NodeProfile: &nodeProfile}
s.testServer = newServerFromIntegrationTestEnv(c)
}
func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
- s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
+ s.testHandler.Cluster.API.MaxItemsPerResponse = 1
req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
arvadostest.QueuedContainerUUID))),
mux.Handle("/_health/", &health.Handler{
Token: h.Cluster.ManagementToken,
Prefix: "/_health/",
+ Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
})
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+func semaphore(max int) (acquire, release func()) {
+ if max > 0 {
+ ch := make(chan bool, max)
+ return func() { ch <- true }, func() { <-ch }
+ } else {
+ return func() {}, func() {}
+ }
+}
"os"
"git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
} else if err != nil {
return 2
}
- cfg, err := arvados.GetConfig(*configFile)
+ cfg, err := config.LoadFile(*configFile, log)
if err != nil {
return 1
}
}
}
-type RequestLimits struct {
- MaxItemsPerResponse int
- MultiClusterRequestConcurrency int
+type API struct {
+ MaxItemsPerResponse int
+ MaxRequestAmplification int
}
type Cluster struct {
HTTPRequestTimeout Duration
RemoteClusters map[string]RemoteCluster
PostgreSQL PostgreSQL
- RequestLimits RequestLimits
+ API API
Logging Logging
TLS TLS
}
Keepbalance Service
Keepproxy Service
Keepstore Service
- Keepweb Service
Nodemanager Service
RailsAPI Service
+ WebDAV Service
Websocket Service
- Workbench Service
+ Workbench1 Service
+ Workbench2 Service
}
type Service struct {
return err
}
+func (su URL) MarshalText() ([]byte, error) {
+ return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
+}
+
type ServiceInstance struct{}
type Logging struct {
}
}
-func (h RequestLimits) GetMultiClusterRequestConcurrency() int {
- if h.MultiClusterRequestConcurrency == 0 {
- return 4
- }
- return h.MultiClusterRequestConcurrency
-}
-
-func (h RequestLimits) GetMaxItemsPerResponse() int {
- if h.MaxItemsPerResponse == 0 {
- return 1000
- }
- return h.MaxItemsPerResponse
-}
-
type SystemServiceInstance struct {
Listen string
TLS bool
func (c PostgreSQLConnection) String() string {
s := ""
for k, v := range c {
+ if v == "" {
+ continue
+ }
s += strings.ToLower(k)
s += "='"
s += strings.Replace(
f.write("""
Clusters:
zzzzz:
+ ManagementToken: e687950a23c3a9bceec28c6223a06c79
HTTPRequestTimeout: 30s
PostgreSQL:
ConnectionPool: 32
Connection:
- host: {}
- dbname: {}
- user: {}
- password: {}
+ Host: {}
+ DBName: {}
+ User: {}
+ Password: {}
NodeProfiles:
"*":
"arvados-controller":
agh = subprocess.Popen(
['arv-git-httpd',
'-repo-root='+gitdir+'/test',
+ '-management-token=e687950a23c3a9bceec28c6223a06c79',
'-address=:'+str(gitport)],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('arv-git-httpd'), 'w') as f:
['keep-web',
'-allow-anonymous',
'-attachment-only-host=download',
+ '-management-token=e687950a23c3a9bceec28c6223a06c79',
'-listen=:'+str(keepwebport)],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('keep-web'), 'w') as f:
gem 'rails-perftest'
gem 'rails-controller-testing'
+gem 'sass-rails'
+
# Install any plugin gems
Dir.glob(File.join(File.dirname(__FILE__), 'lib', '**', "Gemfile")) do |f|
eval(IO.read(f), binding)
faye-websocket (0.10.7)
eventmachine (>= 0.12.0)
websocket-driver (>= 0.5.1)
+ ffi (1.9.25)
globalid (0.4.2)
activesupport (>= 4.2.0)
googleauth (0.8.0)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
rake (12.3.2)
+ rb-fsevent (0.10.3)
+ rb-inotify (0.9.10)
+ ffi (>= 0.5.0, < 2)
ref (2.0.0)
request_store (1.4.1)
rack (>= 1.4)
rvm-capistrano (1.5.6)
capistrano (~> 2.15.4)
safe_yaml (1.0.5)
+ sass (3.5.5)
+ sass-listen (~> 4.0.0)
+ sass-listen (4.0.0)
+ rb-fsevent (~> 0.9, >= 0.9.4)
+ rb-inotify (~> 0.9, >= 0.9.7)
+ sass-rails (5.0.7)
+ railties (>= 4.0.0, < 6)
+ sass (~> 3.1)
+ sprockets (>= 2.8, < 4.0)
+ sprockets-rails (>= 2.0, < 4.0)
+ tilt (>= 1.1, < 3)
signet (0.11.0)
addressable (~> 2.3)
faraday (~> 0.9)
ref
thor (0.20.3)
thread_safe (0.3.6)
+ tilt (2.0.8)
tzinfo (1.2.5)
thread_safe (~> 0.1)
uglifier (2.7.2)
ruby-prof (~> 0.15.0)
rvm-capistrano
safe_yaml
+ sass-rails
simplecov (~> 0.7.1)
simplecov-rcov
sshkey
* and any sub-directories. You're free to add application-wide styles to this file and they'll appear at
* the top of the compiled file, but it's generally better to create a new file per style scope.
*= require_self
- *= require_tree .
+ *= require_tree .
*/
.contain-align-left {
font-size: 0.8em;
}
img.curoverse-logo {
- width: 221px;
- height: 44px;
+ height: 66px;
}
#intropage {
font-family: Verdana,Arial,sans-serif;
color: #000;
font-weight: bold;
}
-
Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
remoteHosts = {}
- Rails.configuration.RemoteClusters.each {|k,v| if k != "*" then remoteHosts[k] = v["Host"] end }
+ Rails.configuration.RemoteClusters.each {|k,v| if k != :"*" then remoteHosts[k] = v["Host"] end }
discovery = {
kind: "discovery#restDescription",
discoveryVersion: "v1",
remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"].Proxy,
websocketUrl: Rails.configuration.Services.Websocket.ExternalURL.to_s,
workbenchUrl: Rails.configuration.Services.Workbench1.ExternalURL.to_s,
+ workbench2Url: Rails.configuration.Services.Workbench2.ExternalURL.to_s,
keepWebServiceUrl: Rails.configuration.Services.WebDAV.ExternalURL.to_s,
gitUrl: Rails.configuration.Services.GitHTTP.ExternalURL.to_s,
parameters: {
end
def merge
- if !Thread.current[:api_client].andand.is_trusted
- return send_error("supplied API token is not from a trusted client", status: 403)
- elsif Thread.current[:api_client_authorization].scopes != ['all']
- return send_error("cannot merge with a scoped token", status: 403)
- end
+ if (params[:old_user_uuid] || params[:new_user_uuid])
+ if !current_user.andand.is_admin
+ return send_error("Must be admin to use old_user_uuid/new_user_uuid", status: 403)
+ end
+ if !params[:old_user_uuid] || !params[:new_user_uuid]
+ return send_error("Must supply both old_user_uuid and new_user_uuid", status: 422)
+ end
+ new_user = User.find_by_uuid(params[:new_user_uuid])
+ if !new_user
+ return send_error("User in new_user_uuid not found", status: 422)
+ end
+ @object = User.find_by_uuid(params[:old_user_uuid])
+ if !@object
+ return send_error("User in old_user_uuid not found", status: 422)
+ end
+ else
+ if !Thread.current[:api_client].andand.is_trusted
+ return send_error("supplied API token is not from a trusted client", status: 403)
+ elsif Thread.current[:api_client_authorization].scopes != ['all']
+ return send_error("cannot merge with a scoped token", status: 403)
+ end
- new_auth = ApiClientAuthorization.validate(token: params[:new_user_token])
- if !new_auth
- return send_error("invalid new_user_token", status: 401)
- end
- if !new_auth.api_client.andand.is_trusted
- return send_error("supplied new_user_token is not from a trusted client", status: 403)
- elsif new_auth.scopes != ['all']
- return send_error("supplied new_user_token has restricted scope", status: 403)
+ new_auth = ApiClientAuthorization.validate(token: params[:new_user_token])
+ if !new_auth
+ return send_error("invalid new_user_token", status: 401)
+ end
+
+ if new_auth.user.uuid[0..4] == Rails.configuration.ClusterID
+ if !new_auth.api_client.andand.is_trusted
+ return send_error("supplied new_user_token is not from a trusted client", status: 403)
+ elsif new_auth.scopes != ['all']
+ return send_error("supplied new_user_token has restricted scope", status: 403)
+ end
+ end
+ new_user = new_auth.user
+ @object = current_user
end
- new_user = new_auth.user
- if current_user.uuid == new_user.uuid
+ if @object.uuid == new_user.uuid
return send_error("cannot merge user to self", status: 422)
end
+ if !params[:new_owner_uuid]
+ return send_error("missing new_owner_uuid", status: 422)
+ end
+
if !new_user.can?(write: params[:new_owner_uuid])
return send_error("cannot move objects into supplied new_owner_uuid: new user does not have write permission", status: 403)
end
redirect = params[:redirect_to_new_user]
+ if @object.uuid[0..4] != Rails.configuration.ClusterID && redirect
+ return send_error("cannot merge remote user to other with redirect_to_new_user=true", status: 422)
+ end
+
if !redirect
return send_error("merge with redirect_to_new_user=false is not yet supported", status: 422)
end
- @object = current_user
act_as_system_user do
@object.merge(new_owner_uuid: params[:new_owner_uuid], redirect_to_user_uuid: redirect && new_user.uuid)
end
type: 'string', required: true,
},
new_user_token: {
- type: 'string', required: true,
+ type: 'string', required: false,
},
redirect_to_new_user: {
type: 'boolean', required: false,
},
+ old_user_uuid: {
+ type: 'string', required: false,
+ },
+ new_user_uuid: {
+ type: 'string', required: false,
+ }
}
end
# For the benefit of functional and integration tests:
@user = user
+ if user.uuid[0..4] != Rails.configuration.ClusterID
+ # Actually a remote user
+ # Send them to their home cluster's login
+ rh = Rails.configuration.RemoteClusters[user.uuid[0..4]]
+ remote, return_to_url = params[:return_to].split(',', 2)
+ @remotehomeurl = "#{rh.Scheme || "https"}://#{rh.Host}/login?remote=#{Rails.configuration.ClusterID}&return_to=#{return_to_url}"
+ render
+ return
+ end
+
# prevent ArvadosModel#before_create and _update from throwing
# "unauthorized":
Thread.current[:user] = user
# This container is finished so finalize any associated container requests
# that are associated with this container.
if self.state_changed? and self.final?
- act_as_system_user do
-
- if self.state == Cancelled
- retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid)
- else
- retryable_requests = []
- end
+ # These get wiped out by with_lock (which reloads the record),
+ # so record them now in case we need to schedule a retry.
+ prev_secret_mounts = self.secret_mounts_was
+ prev_runtime_token = self.runtime_token_was
+
+ # Need to take a lock on the container to ensure that any
+ # concurrent container requests that might try to reuse this
+ # container will block until the container completion
+ # transaction finishes. This ensure that concurrent container
+ # requests that try to reuse this container are finalized (on
+ # Complete) or don't reuse it (on Cancelled).
+ self.with_lock do
+ act_as_system_user do
+ if self.state == Cancelled
+ retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid)
+ else
+ retryable_requests = []
+ end
- if retryable_requests.any?
- c_attrs = {
- command: self.command,
- cwd: self.cwd,
- environment: self.environment,
- output_path: self.output_path,
- container_image: self.container_image,
- mounts: self.mounts,
- runtime_constraints: self.runtime_constraints,
- scheduling_parameters: self.scheduling_parameters,
- secret_mounts: self.secret_mounts_was,
- runtime_token: self.runtime_token_was,
- runtime_user_uuid: self.runtime_user_uuid,
- runtime_auth_scopes: self.runtime_auth_scopes
- }
- c = Container.create! c_attrs
- retryable_requests.each do |cr|
- cr.with_lock do
- leave_modified_by_user_alone do
- # Use row locking because this increments container_count
- cr.container_uuid = c.uuid
- cr.save!
+ if retryable_requests.any?
+ c_attrs = {
+ command: self.command,
+ cwd: self.cwd,
+ environment: self.environment,
+ output_path: self.output_path,
+ container_image: self.container_image,
+ mounts: self.mounts,
+ runtime_constraints: self.runtime_constraints,
+ scheduling_parameters: self.scheduling_parameters,
+ secret_mounts: prev_secret_mounts,
+ runtime_token: prev_runtime_token,
+ runtime_user_uuid: self.runtime_user_uuid,
+ runtime_auth_scopes: self.runtime_auth_scopes
+ }
+ c = Container.create! c_attrs
+ retryable_requests.each do |cr|
+ cr.with_lock do
+ leave_modified_by_user_alone do
+ # Use row locking because this increments container_count
+ cr.container_uuid = c.uuid
+ cr.save!
+ end
end
end
end
- end
- # Notify container requests associated with this container
- ContainerRequest.where(container_uuid: uuid,
- state: ContainerRequest::Committed).each do |cr|
- leave_modified_by_user_alone do
- cr.finalize!
+ # Notify container requests associated with this container
+ ContainerRequest.where(container_uuid: uuid,
+ state: ContainerRequest::Committed).each do |cr|
+ leave_modified_by_user_alone do
+ cr.finalize!
+ end
end
- end
- # Cancel outstanding container requests made by this container.
- ContainerRequest.
- includes(:container).
- where(requesting_container_uuid: uuid,
- state: ContainerRequest::Committed).each do |cr|
- leave_modified_by_user_alone do
- cr.update_attributes!(priority: 0)
- cr.container.reload
- if cr.container.state == Container::Queued || cr.container.state == Container::Locked
- # If the child container hasn't started yet, finalize the
- # child CR now instead of leaving it "on hold", i.e.,
- # Queued with priority 0. (OTOH, if the child is already
- # running, leave it alone so it can get cancelled the
- # usual way, get a copy of the log collection, etc.)
- cr.update_attributes!(state: ContainerRequest::Final)
+ # Cancel outstanding container requests made by this container.
+ ContainerRequest.
+ includes(:container).
+ where(requesting_container_uuid: uuid,
+ state: ContainerRequest::Committed).each do |cr|
+ leave_modified_by_user_alone do
+ cr.update_attributes!(priority: 0)
+ cr.container.reload
+ if cr.container.state == Container::Queued || cr.container.state == Container::Locked
+ # If the child container hasn't started yet, finalize the
+ # child CR now instead of leaving it "on hold", i.e.,
+ # Queued with priority 0. (OTOH, if the child is already
+ # running, leave it alone so it can get cancelled the
+ # usual way, get a copy of the log collection, etc.)
+ cr.update_attributes!(state: ContainerRequest::Final)
+ end
end
end
end
end
def finalize_if_needed
- if state == Committed && Container.find_by_uuid(container_uuid).final?
- reload
- act_as_system_user do
- leave_modified_by_user_alone do
- finalize!
+ return if state != Committed
+ while true
+ # get container lock first, then lock current container request
+ # (same order as Container#handle_completed). Locking always
+ # reloads the Container and ContainerRequest records.
+ c = Container.find_by_uuid(container_uuid)
+ c.lock!
+ self.lock!
+
+ if container_uuid != c.uuid
+ # After locking, we've noticed a race, the container_uuid is
+ # different than the container record we just loaded. This
+ # can happen if Container#handle_completed scheduled a new
+ # container for retry and set container_uuid while we were
+ # waiting on the container lock. Restart the loop and get the
+ # new container.
+ redo
+ end
+
+ if state == Committed && c.final?
+ # The current container is
+ act_as_system_user do
+ leave_modified_by_user_alone do
+ finalize!
+ end
end
end
+ return true
end
end
self.mounts ||= {}
self.secret_mounts ||= {}
self.cwd ||= "."
- self.container_count_max ||= Rails.configuration.Containers.MaxComputeVMs
+ self.container_count_max ||= Rails.configuration.Containers.MaxRetryAttempts
self.scheduling_parameters ||= {}
self.output_ttl ||= 0
self.priority ||= 0
return false
end
if state_changed? and state == Committed and container_uuid.nil?
- self.container_uuid = Container.resolve(self).uuid
+ while true
+ c = Container.resolve(self)
+ c.lock!
+ if c.state == Container::Cancelled
+ # Lost a race, we have a lock on the container but the
+ # container was cancelled in a different request, restart
+ # the loop and resolve request to a new container.
+ redo
+ end
+ self.container_uuid = c.uuid
+ break
+ end
end
if self.container_uuid != self.container_uuid_was
if self.container_count_changed?
<!DOCTYPE html>
<html>
<head>
- <title>Server</title>
+ <title>Arvados API Server (<%= Rails.configuration.ClusterID %>)</title>
<%= stylesheet_link_tag "application" %>
<%= javascript_include_tag "application" %>
<%= csrf_meta_tags %>
</head>
<body>
<div id="header">
- <div class="apptitle">ARVADOS <span class="beta"><span>BETA</span></span></div>
+ <div class="apptitle">ARVADOS</div>
+ <div>(<%= Rails.configuration.ClusterID %>)</div>
<div style="float:right">
<% if current_user %>
<%= current_user.full_name %>
•
<a class="logout" href="/logout">Log out</a>
<% else %>
- <a class="logout" href="/auth/joshid">Log in</a>
+ <!--<a class="logout" href="/auth/joshid">Log in</a>-->
<% end %>
<% if current_user and session[:real_uid] and session[:switch_back_to] and User.find(session[:real_uid].to_i).verify_userswitch_cookie(session[:switch_back_to]) %>
<p>Sorry, something went wrong logging you in. Please try again.</p>
- <p style="float:right;margin-top:1em">
- <a href="/auth/joshid">Log in here.</a>
- </p>
+ <!--<p style="float:right;margin-top:1em">
+ <a href="/login">Log in here.</a>
+ </p>-->
<div style="clear:both;height:8em"></div>
</div>
--- /dev/null
+<div style="width:30em; margin:2em auto 0 auto">
+ <h1>Login redirect</h1>
+ <p>This login is linked to federated user <b><%= @user.uuid %></b> on cluster <b><%= @user.uuid[0..4] %></b>.</p>
+ <p><a href="<%=@remotehomeurl%>">Click here log in on cluster <%= @user.uuid[0..4] %>.</a></p>
+ <p>After logging in, you will be returned to this cluster (<%=Rails.configuration.ClusterID%>).</p>
+ <p>To avoid seeing this page, choose <b><%= @user.uuid[0..4] %></b> as the cluster that hosts your user account on the Workbench login page.</p>
+</div>
arvcfg.declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
arvcfg.declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
arvcfg.declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
-arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
+arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Collections.BlobSigning", !v }
arvcfg.declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
arvcfg.declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
arvcfg.declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
dbcfg.declare_config "PostgreSQL.ConnectionPool", Integer, :pool
dbcfg.declare_config "PostgreSQL.Connection.Host", String, :host
-dbcfg.declare_config "PostgreSQL.Connection.Port", Integer, :port
+dbcfg.declare_config "PostgreSQL.Connection.Port", String, :port
dbcfg.declare_config "PostgreSQL.Connection.User", String, :username
dbcfg.declare_config "PostgreSQL.Connection.Password", String, :password
dbcfg.declare_config "PostgreSQL.Connection.DBName", String, :database
combined.update $remaining_config
puts combined.to_yaml
end
+
+ desc 'Legacy config check task -- it is a noop now'
+ task check: :environment do
+ # This exists so that build/rails-package-scripts/postinst.sh doesn't fail.
+ end
end
redirect_to_new_user: true,
})
assert_response(:success)
- assert_equal(users(:project_viewer).redirect_to_user_uuid, users(:active).uuid)
+ assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
auth = ApiClientAuthorization.validate(token: api_client_authorizations(:project_viewer).api_token)
assert_not_nil(auth)
assert_equal(users(:active).uuid, auth.user.uuid)
end
+
+ test "merge 'project_viewer' account into 'active' account using uuids" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(:success)
+ assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
+
+ auth = ApiClientAuthorization.validate(token: api_client_authorizations(:project_viewer).api_token)
+ assert_not_nil(auth)
+ assert_not_nil(auth.user)
+ assert_equal(users(:active).uuid, auth.user.uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied for non-admin" do
+ authorize_with(:active)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(403)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied missing old_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied missing new_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied bogus old_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: "zzzzz-tpzed-bogusbogusbogus",
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied bogus new_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: "zzzzz-tpzed-bogusbogusbogus",
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
"last_name", "username"].sort
fi, err := os.Stat(dev)
if err != nil {
- return giveup("stat %q: %s\n", dev, err)
+ return giveup("stat %q: %s", dev, err)
}
ino := fi.Sys().(*syscall.Stat_t).Ino
n, err := io.Copy(tmpfile, rdr)
v.os.stats.TickOutBytes(uint64(n))
if err != nil {
- log.Printf("%s: writing to %s: %s\n", v, bpath, err)
+ log.Printf("%s: writing to %s: %s", v, bpath, err)
tmpfile.Close()
v.os.Remove(tmpfile.Name())
return err
}
if err := tmpfile.Close(); err != nil {
- log.Printf("closing %s: %s\n", tmpfile.Name(), err)
+ log.Printf("closing %s: %s", tmpfile.Name(), err)
v.os.Remove(tmpfile.Name())
return err
}
if err := v.os.Rename(tmpfile.Name(), bpath); err != nil {
- log.Printf("rename %s %s: %s\n", tmpfile.Name(), bpath, err)
+ log.Printf("rename %s %s: %s", tmpfile.Name(), bpath, err)
return v.os.Remove(tmpfile.Name())
}
return nil
func (v *UnixVolume) Status() *VolumeStatus {
fi, err := v.os.Stat(v.Root)
if err != nil {
- log.Printf("%s: os.Stat: %s\n", v, err)
+ log.Printf("%s: os.Stat: %s", v, err)
return nil
}
devnum := fi.Sys().(*syscall.Stat_t).Dev
var fs syscall.Statfs_t
if err := syscall.Statfs(v.Root, &fs); err != nil {
- log.Printf("%s: statfs: %s\n", v, err)
+ log.Printf("%s: statfs: %s", v, err)
return nil
}
// These calculations match the way df calculates disk usage:
if avail, err := v.FreeDiskSpace(); err == nil {
isFull = avail < MinFreeKilobytes
} else {
- log.Printf("%s: FreeDiskSpace: %s\n", v, err)
+ log.Printf("%s: FreeDiskSpace: %s", v, err)
isFull = false
}
if v.locker == nil {
return nil
}
+ t0 := time.Now()
locked := make(chan struct{})
go func() {
v.locker.Lock()
}()
select {
case <-ctx.Done():
+ log.Printf("%s: client hung up while waiting for Serialize lock (%s)", v, time.Since(t0))
go func() {
<-locked
v.locker.Unlock()
# size class (since libcloud does not provide any consistent API for exposing
# this setting).
# You may also want to define the amount of scratch space (expressed
-# in GB) for Crunch jobs. You can also override Amazon's provided
+# in MB) for Crunch jobs. You can also override Amazon's provided
# data fields (such as price per hour) by setting them here.
#
# Additionally, you can ask for a preemptible instance (AWS's spot instance)
# both spot & reserved versions of the same size, you can do so by renaming
# the Size section and specifying the instance type inside it.
+# 100 GB scratch space
[Size m4.large]
cores = 2
price = 0.126
-scratch = 100
+scratch = 100000
+# 10 GB scratch space
[Size m4.large.spot]
instance_type = m4.large
preemptible = true
cores = 2
price = 0.126
-scratch = 100
+scratch = 10000
+# 200 GB scratch space
[Size m4.xlarge]
cores = 4
price = 0.252
-scratch = 100
+scratch = 200000
'apache-libcloud>=2.3.1.dev1',
'arvados-python-client>=0.1.20170731145219',
'future',
- 'pykka',
+ 'pykka < 2',
'python-daemon',
'setuptools',
'subprocess32>=3.5.1',
;;
root-cert)
- CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.pem
+ CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt
if test -n "$1" ; then
CERT="$1"
fi
. /usr/local/lib/arvbox/common.sh
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+
if test ! -s /var/lib/arvados/root-cert.pem ; then
# req signing request sub-command
# -new new certificate request
-nodes \
-sha256 \
-x509 \
- -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=arvbox testing root CA for ${uuid_prefix}" \
+ -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=test root CA for ${uuid_prefix} generated $(date --rfc-3339=seconds)" \
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nbasicConstraints=critical,CA:true,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign")) \
-new \
-nodes \
-sha256 \
- -subj "/C=US/ST=MA/O=Arvados testing for ${uuid_prefix}/OU=arvbox/CN=localhost" \
+ -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=test server cert for ${uuid_prefix} generated $(date --rfc-3339=seconds)" \
-reqexts x509_ext \
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
fi
uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+secret_token=$(cat /var/lib/arvados/api_secret_token)
+blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+management_token=$(cat /var/lib/arvados/management_token)
+sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
+vm_uuid=$(cat /var/lib/arvados/vm-uuid)
database_pw=$(cat /var/lib/arvados/api_database_pw)
if test -s /var/lib/arvados/api_rails_env ; then
cat >/var/lib/arvados/cluster_config.yml <<EOF
Clusters:
${uuid_prefix}:
- NodeProfiles:
+ ManagementToken: $management_token
+ Services:
+ Workbench1:
+ ExternalURL: "https://$localip:${services[workbench]}"
+ Workbench2:
+ ExternalURL: "https://$localip:${services[workbench2-ssl]}"
+ SSO:
+ ExternalURL: "https://$localip:${services[sso]}"
+ Websocket:
+ ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
+ GitSSH:
+ ExternalURL: "ssh://git@$localip:"
+ GitHTTP:
+ ExternalURL: "http://$localip:${services[arv-git-httpd]}/"
+ WebDAV:
+ ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
+ NodeProfiles: # to be deprecated in favor of "Services" section
"*":
arvados-controller:
Listen: ":${services[controller]}" # choose a port
Password: ${database_pw}
DBName: arvados_${database_env}
client_encoding: utf8
+ API:
+ RailsSessionSecretToken: $secret_token
+ Collections:
+ BlobSigningKey: $blob_signing_key
+ DefaultReplication: 1
+ Containers:
+ SupportedDockerImageFormats: ["v2"]
+ Login:
+ ProviderAppSecret: $sso_app_secret
+ ProviderAppID: arvados-server
+ Users:
+ NewUsersAreActive: true
+ AutoAdminFirstUser: true
+ AutoSetupNewUsers: true
+ AutoSetupNewUsersWithVmUUID: $vm_uuid
+ AutoSetupNewUsersWithRepository: true
EOF
/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
cat > /usr/local/bin/crunch-run.sh <<EOF
#!/bin/sh
-exec /usr/local/bin/crunch-run -container-enable-networking=always -container-network-mode=host \$@
+exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
EOF
chmod +x /usr/local/bin/crunch-run.sh
}
EOF
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+url_prefix="https://$localip:${services[workbench2-ssl]}/"
+
+set +e
+read -rd $'\000' apiclient <<EOF
+{
+ "url_prefix": "$url_prefix",
+ "is_trusted": true
+}
+EOF
+set -e
+
+clientuuid=$(arv --format=uuid api_client list --filters '[["url_prefix", "=", "'$url_prefix'"]]')
+if [[ -n "$clientuuid" ]] ; then
+ arv api_client update --uuid $clientuuid --api-client "$apiclient"
+else
+ arv api_client create --api-client "$apiclient"
+fi
+
export HTTPS=false
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+#
+
+from __future__ import print_function, absolute_import
+import argparse
+import arvados
+import arvados.util
+import csv
+import sys
+import logging
+
+lglvl = logging.INFO+1
+logging.basicConfig(level=lglvl, format='%(message)s')
+
+"""
+ Given a list of collections missing blocks (as produced by
+keep-balance), produce a report listing affected collections and
+container requests.
+"""
+
+def rerun_request(arv, container_requests_to_rerun, ct):
+ requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
+ for cr in requests:
+ if cr["requesting_container_uuid"]:
+ rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
+ else:
+ container_requests_to_rerun[cr["uuid"]] = cr
+
+def get_owner(arv, owners, record):
+ uuid = record["owner_uuid"]
+ if uuid not in owners:
+ if uuid[6:11] == "tpzed":
+ owners[uuid] = (arv.users().get(uuid=uuid).execute()["full_name"], uuid)
+ else:
+ grp = arv.groups().get(uuid=uuid).execute()
+ _, ou = get_owner(arv, owners, grp)
+ owners[uuid] = (grp["name"], ou)
+ return owners[uuid]
+
+def main():
+ parser = argparse.ArgumentParser(description='Re-run containers associated with missing blocks')
+ parser.add_argument('inp')
+ args = parser.parse_args()
+
+ arv = arvados.api('v1')
+
+ busted_collections = set()
+
+ logging.log(lglvl, "Reading %s", args.inp)
+
+ # Get the list of bad collection PDHs
+ blocksfile = open(args.inp, "rt")
+ for line in blocksfile:
+ # Ignore the first item, that's the block id
+ collections = line.rstrip().split(" ")[1:]
+ for c in collections:
+ busted_collections.add(c)
+
+ out = csv.writer(sys.stdout)
+
+ out.writerow(("collection uuid", "container request uuid", "record name", "modified at", "owner uuid", "owner name", "root owner uuid", "root owner name", "notes"))
+
+ logging.log(lglvl, "Finding collections")
+
+ owners = {}
+ collections_to_delete = {}
+ container_requests_to_rerun = {}
+ # Get containers that produced these collections
+ i = 0
+ for b in busted_collections:
+ if (i % 100) == 0:
+ logging.log(lglvl, "%d/%d", i, len(busted_collections))
+ i += 1
+ collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
+ for d in collections_to_delete:
+ t = ""
+ if d["properties"].get("type") not in ("output", "log"):
+ t = "\"type\" was '%s', expected one of 'output' or 'log'" % d["properties"].get("type")
+ ou = get_owner(arv, owners, d)
+ out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], t))
+
+ maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
+ for ct in maybe_containers_to_rerun:
+ rerun_request(arv, container_requests_to_rerun, ct)
+
+ logging.log(lglvl, "%d/%d", i, len(busted_collections))
+ logging.log(lglvl, "Finding container requests")
+
+ i = 0
+ for _, cr in container_requests_to_rerun.items():
+ if (i % 100) == 0:
+ logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
+ i += 1
+ ou = get_owner(arv, owners, cr)
+ out.writerow(("", cr["uuid"], cr["name"], cr["modified_at"], cr["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], ""))
+
+ logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
+
+if __name__ == "__main__":
+ main()
"revision": "0a025b7e63adc15a622f29b0b2c4c3848243bbf6",
"revisionTime": "2016-08-13T22:13:03Z"
},
+ {
+ "checksumSHA1": "x7IEwuVYTztOJItr3jtePGyFDWA=",
+ "path": "github.com/imdario/mergo",
+ "revision": "5ef87b449ca75fbed1bc3765b749ca8f73f1fa69",
+ "revisionTime": "2019-04-15T13:31:43Z"
+ },
{
"checksumSHA1": "iCsyavJDnXC9OY//p52IWJWy7PY=",
"path": "github.com/jbenet/go-context/io",
projects / arvados.git / commitdiff
