14717: Make CONFIGSRC mandatory for run-tests.sh

Must point to a directory containing a config.yml with a cluster id of
'zzzzz' and a PostgreSQL section.

Includes a few other cleanups.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9bcbe4d93be31563879e8cf2bd66af0e88e40f4b..6d73ee9908a75e8e5cf17ce8b73afebf5eef9938 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -35,8 +35,7 @@ Options:
 --short        Skip (or scale down) some slow tests.
 --interactive  Set up, then prompt for test/install steps to perform.
 WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with api server config files to copy into source tree.
-               (If none given, leave config files alone in source tree.)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
 services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
                Restrict apiserver tests to the given file
 sdk/python_test="--test-suite tests.test_keep_locator"
@@ -197,6 +196,8 @@ sanity_checks() {
     [[ -n "${skip[sanity]}" ]] && return 0
     ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
         || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+    ( [[ -n "$CONFIGSRC" ]] && [[ -s "$CONFIGSRC/config.yml" ]] ) \
+        || fatal "CONFIGSRC environment variable not set to a config file (see: $0 --help)"
     echo Checking dependencies:
     echo "locale: ${LANG}"
     [[ "$(locale charmap)" = "UTF-8" ]] \
@@ -576,11 +577,6 @@ initialize() {
 
     echo "WORKSPACE=$WORKSPACE"
 
-    if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
-        # Jenkins expects us to use this by default.
-        CONFIGSRC="$HOME/arvados-api-server"
-    fi
-
     # Clean up .pyc files that may exist in the workspace
     cd "$WORKSPACE"
     find -name '*.pyc' -delete

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 81c1b841230376db1d995058124444d47b314063..0f8f1c5f818e2768168e4c71094a67c34fb4478e 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -677,55 +677,56 @@ def setup_config():
     keep_web_dl_port = find_available_port()
     keep_web_dl_external_port = find_available_port()
 
-    if "CONFIGSRC" in os.environ:
-        dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
-    else:
-        dbconf = "/etc/arvados/config.yml"
+    dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
 
     print("Getting config from %s" % dbconf, file=sys.stderr)
 
-    pgconnection = list(yaml.safe_load(open(dbconf))["Clusters"].values())[0]["PostgreSQL"]["Connection"]
-
-    if "test" not in pgconnection["dbname"]:
-        pgconnection["dbname"] = "arvados_test"
+    pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
 
     localhost = "127.0.0.1"
     services = {
         "RailsAPI": {
-            "InternalURLs": { }
+            "InternalURLs": {
+                "https://%s:%s"%(localhost, rails_api_port): {}
+            }
         },
         "Controller": {
             "ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, controller_port): {}
+            }
         },
         "Websocket": {
             "ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, websocket_port): {}
+            }
         },
         "GitHTTP": {
             "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, git_httpd_port): {}
+            }
         },
         "Keepproxy": {
             "ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keepproxy_port): {}
+            }
         },
         "WebDAV": {
             "ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keep_web_port): {}
+            }
         },
         "WebDAVDownload": {
             "ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
-            "InternalURLs": { }
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, keep_web_dl_port): {}
+            }
         }
     }
-    services["RailsAPI"]["InternalURLs"]["https://%s:%s"%(localhost, rails_api_port)] = {}
-    services["Controller"]["InternalURLs"]["http://%s:%s"%(localhost, controller_port)] = {}
-    services["Websocket"]["InternalURLs"]["http://%s:%s"%(localhost, websocket_port)] = {}
-    services["GitHTTP"]["InternalURLs"]["http://%s:%s"%(localhost, git_httpd_port)] = {}
-    services["Keepproxy"]["InternalURLs"]["http://%s:%s"%(localhost, keepproxy_port)] = {}
-    services["WebDAV"]["InternalURLs"]["http://%s:%s"%(localhost, keep_web_port)] = {}
-    services["WebDAVDownload"]["InternalURLs"]["http://%s:%s"%(localhost, keep_web_dl_port)] = {}
 
     config = {
         "Clusters": {
@@ -754,7 +755,6 @@ def setup_config():
         yaml.safe_dump(config, f)
 
     ex = "export ARVADOS_CONFIG="+conf
-    print(ex, file=sys.stderr)
     print(ex)
 
 

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1a7ad6fac745067fa9da535ba3052321f062e9de..9f69c44460caf7e03733acf619817f31e3a72327 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -24,8 +24,8 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/dispatch"
        "github.com/coreos/go-systemd/daemon"
+       "github.com/ghodss/yaml"
        "github.com/sirupsen/logrus"
-       "gopkg.in/yaml.v2"
 )
 
 type logger interface {

diff --git a/services/ws/main.go b/services/ws/main.go
index 0556c77d610d5dede9112b419803f06227992007..de8e6328dbaadb0ba5568a229f6e62193ecd28cb 100644 (file)
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -12,8 +12,8 @@ import (
        "git.curoverse.com/arvados.git/lib/config"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+       "github.com/ghodss/yaml"
        "github.com/sirupsen/logrus"
-       "gopkg.in/yaml.v2"
 )
 
 var logger = ctxlog.FromContext

diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 1b9a50ca6742c85978ce8a512117070995501c11..7eef27258b58b1c2b9a3d81b8ece6f4b35ffa019 100644 (file)
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -7,7 +7,6 @@ package main
 import (
        "encoding/json"
        "io/ioutil"
-       "log"
        "net/http"
        "os"
        "sync"
@@ -163,16 +162,16 @@ ManagementToken: qqqqq
 `)
        tmpfile, err := ioutil.TempFile("", "example")
        if err != nil {
-               log.Fatal(err)
+               c.Error(err)
        }
 
        defer os.Remove(tmpfile.Name()) // clean up
 
        if _, err := tmpfile.Write(content); err != nil {
-               log.Fatal(err)
+               c.Error(err)
        }
        if err := tmpfile.Close(); err != nil {
-               log.Fatal(err)
+               c.Error(err)
 
        }
        cluster := configure(logger(nil), []string{"arvados-ws", "-config", tmpfile.Name()})