From: Peter Amstutz Date: Mon, 29 Jul 2019 15:38:13 +0000 (-0400) Subject: 14717: Make CONFIGSRC mandatory for run-tests.sh X-Git-Tag: 2.0.0~241^2~8 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/ecbb701177f9ae8713d3e103b3014f10fe584025 14717: Make CONFIGSRC mandatory for run-tests.sh Must point to a directory containing a config.yml with a cluster id of 'zzzzz' and a PostgreSQL section. Includes a few other cleanups. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/build/run-tests.sh b/build/run-tests.sh index 9bcbe4d93b..6d73ee9908 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -35,8 +35,7 @@ Options: --short Skip (or scale down) some slow tests. --interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. -CONFIGSRC=path Dir with api server config files to copy into source tree. - (If none given, leave config files alone in source tree.) +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required) services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file sdk/python_test="--test-suite tests.test_keep_locator" @@ -197,6 +196,8 @@ sanity_checks() { [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" + ( [[ -n "$CONFIGSRC" ]] && [[ -s "$CONFIGSRC/config.yml" ]] ) \ + || fatal "CONFIGSRC environment variable not set to a config file (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ @@ -576,11 +577,6 @@ initialize() { echo "WORKSPACE=$WORKSPACE" - if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then - # Jenkins expects us to use this by default. - CONFIGSRC="$HOME/arvados-api-server" - fi - # Clean up .pyc files that may exist in the workspace cd "$WORKSPACE" find -name '*.pyc' -delete diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 81c1b84123..0f8f1c5f81 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -677,55 +677,56 @@ def setup_config(): keep_web_dl_port = find_available_port() keep_web_dl_external_port = find_available_port() - if "CONFIGSRC" in os.environ: - dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml") - else: - dbconf = "/etc/arvados/config.yml" + dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml") print("Getting config from %s" % dbconf, file=sys.stderr) - pgconnection = list(yaml.safe_load(open(dbconf))["Clusters"].values())[0]["PostgreSQL"]["Connection"] - - if "test" not in pgconnection["dbname"]: - pgconnection["dbname"] = "arvados_test" + pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"] localhost = "127.0.0.1" services = { "RailsAPI": { - "InternalURLs": { } + "InternalURLs": { + "https://%s:%s"%(localhost, rails_api_port): {} + } }, "Controller": { "ExternalURL": "https://%s:%s" % (localhost, controller_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, controller_port): {} + } }, "Websocket": { "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, websocket_port): {} + } }, "GitHTTP": { "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, git_httpd_port): {} + } }, "Keepproxy": { "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, keepproxy_port): {} + } }, "WebDAV": { "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, keep_web_port): {} + } }, "WebDAVDownload": { "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port), - "InternalURLs": { } + "InternalURLs": { + "http://%s:%s"%(localhost, keep_web_dl_port): {} + } } } - services["RailsAPI"]["InternalURLs"]["https://%s:%s"%(localhost, rails_api_port)] = {} - services["Controller"]["InternalURLs"]["http://%s:%s"%(localhost, controller_port)] = {} - services["Websocket"]["InternalURLs"]["http://%s:%s"%(localhost, websocket_port)] = {} - services["GitHTTP"]["InternalURLs"]["http://%s:%s"%(localhost, git_httpd_port)] = {} - services["Keepproxy"]["InternalURLs"]["http://%s:%s"%(localhost, keepproxy_port)] = {} - services["WebDAV"]["InternalURLs"]["http://%s:%s"%(localhost, keep_web_port)] = {} - services["WebDAVDownload"]["InternalURLs"]["http://%s:%s"%(localhost, keep_web_dl_port)] = {} config = { "Clusters": { @@ -754,7 +755,6 @@ def setup_config(): yaml.safe_dump(config, f) ex = "export ARVADOS_CONFIG="+conf - print(ex, file=sys.stderr) print(ex) diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go index 1a7ad6fac7..9f69c44460 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go @@ -24,8 +24,8 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/dispatch" "github.com/coreos/go-systemd/daemon" + "github.com/ghodss/yaml" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) type logger interface { diff --git a/services/ws/main.go b/services/ws/main.go index 0556c77d61..de8e6328db 100644 --- a/services/ws/main.go +++ b/services/ws/main.go @@ -12,8 +12,8 @@ import ( "git.curoverse.com/arvados.git/lib/config" "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/ctxlog" + "github.com/ghodss/yaml" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) var logger = ctxlog.FromContext diff --git a/services/ws/server_test.go b/services/ws/server_test.go index 1b9a50ca67..7eef27258b 100644 --- a/services/ws/server_test.go +++ b/services/ws/server_test.go @@ -7,7 +7,6 @@ package main import ( "encoding/json" "io/ioutil" - "log" "net/http" "os" "sync" @@ -163,16 +162,16 @@ ManagementToken: qqqqq `) tmpfile, err := ioutil.TempFile("", "example") if err != nil { - log.Fatal(err) + c.Error(err) } defer os.Remove(tmpfile.Name()) // clean up if _, err := tmpfile.Write(content); err != nil { - log.Fatal(err) + c.Error(err) } if err := tmpfile.Close(); err != nil { - log.Fatal(err) + c.Error(err) } cluster := configure(logger(nil), []string{"arvados-ws", "-config", tmpfile.Name()})