Merge branch '15467-update-install-docs' refs #15467

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 113069a33e29b2fe64eec48b53822908c6070f6b..d70722272cf70f9de8e4ff4a2384b7e89366eb8a 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -654,7 +654,7 @@ install_env() {
     . "$VENVDIR/bin/activate"
 
     # Needed for run_test_server.py which is used by certain (non-Python) tests.
-    pip install --no-cache-dir PyYAML \
+    pip install --no-cache-dir PyYAML future \
         || fatal "pip install PyYAML failed"
 
     # Preinstall libcloud if using a fork; otherwise nodemanager "pip
@@ -819,11 +819,14 @@ do_test_once() {
 }
 
 check_arvados_config() {
+    if [[ "$1" = "env" ]] ; then
+       return
+    fi
     if [[ -z "$ARVADOS_CONFIG" ]] ; then
        # Create config file.  The run_test_server script requires PyYAML,
        # so virtualenv needs to be active.  Downstream steps like
        # workbench install which require a valid config.yml.
-       if [[ (! -s "$VENVDIR/bin/activate") && "$1" != "env" ]] ; then
+       if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
            install_env
        fi
        . "$VENVDIR/bin/activate"
@@ -971,11 +974,15 @@ install_services/api() {
         && git --git-dir internal.git init \
             || return 1
 
-    cd "$WORKSPACE/services/api" \
-        && RAILS_ENV=test bundle exec rails db:environment:set \
-        && RAILS_ENV=test bundle exec rake db:drop \
-        && RAILS_ENV=test bundle exec rake db:setup \
-        && RAILS_ENV=test bundle exec rake db:fixtures:load
+
+    (cd "$WORKSPACE/services/api"
+     export RAILS_ENV=test
+     if bundle exec rails db:environment:set ; then
+        bundle exec rake db:drop
+     fi
+     bundle exec rake db:setup \
+        && bundle exec rake db:fixtures:load
+    )
 }
 
 declare -a pythonstuff

diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index 22364d2ae5897ffbc119172bab4fab4e227d9f67..e53e4980a86f01a595649331d020c6b87e823e6a 100644 (file)
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -199,7 +199,11 @@ class PollClient(threading.Thread):
                             # filter on that same cutoff time, or
                             # (once we see our first matching event)
                             # the ID of the last-seen event.
-                            self._skip_old_events = [[
+                            #
+                            # Note: self._skip_old_events must not be
+                            # set until the threshold is decided.
+                            # Otherwise, tests will be unreliable.
+                            filter_by_time = [[
                                 "created_at", ">=",
                                 time.strftime(
                                     "%Y-%m-%dT%H:%M:%SZ",
@@ -207,7 +211,7 @@ class PollClient(threading.Thread):
                             items = self.api.logs().list(
                                 order="id desc",
                                 limit=1,
-                                filters=f+self._skip_old_events).execute()
+                                filters=f+filter_by_time).execute()
                             if items["items"]:
                                 self._skip_old_events = [
                                     ["id", ">", str(items["items"][0]["id"])]]
@@ -215,6 +219,11 @@ class PollClient(threading.Thread):
                                     "items": [],
                                     "items_available": 0,
                                 }
+                            else:
+                                # No recent events. We can keep using
+                                # the same timestamp threshold until
+                                # we receive our first new event.
+                                self._skip_old_events = filter_by_time
                         else:
                             # In this case, either we know the most
                             # recent matching ID, or we know there

diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index d60f19627b75f79cb22f26b01f5ddd9139b8b175..4ed25e03c05929bdceecd968d494e194500f7959 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -19,46 +19,19 @@ fi
 
 set -u
 
-uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
-
-if ! test -s /var/lib/arvados/api_secret_token ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/api_secret_token
-fi
-secret_token=$(cat /var/lib/arvados/api_secret_token)
-
-if ! test -s /var/lib/arvados/blob_signing_key ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/blob_signing_key
-fi
-blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
-
-if ! test -s /var/lib/arvados/management_token ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/management_token
-fi
-management_token=$(cat /var/lib/arvados/management_token)
-
-sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
-
-if test -s /var/lib/arvados/vm-uuid ; then
-    vm_uuid=$(cat /var/lib/arvados/vm-uuid)
-else
-    vm_uuid=$uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]')
-    echo $vm_uuid > /var/lib/arvados/vm-uuid
-fi
-
-if ! test -f /var/lib/arvados/api_database_pw ; then
-    ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
-fi
-database_pw=$(cat /var/lib/arvados/api_database_pw)
-
-if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
-    psql postgres -c "create user arvados with password '$database_pw'"
-fi
-psql postgres -c "ALTER USER arvados WITH SUPERUSER;"
+flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
 if test -a /usr/src/arvados/services/api/config/arvados_config.rb ; then
     rm -f config/application.yml config/database.yml
-    flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 else
+    uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+    secret_token=$(cat /var/lib/arvados/api_secret_token)
+    blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+    management_token=$(cat /var/lib/arvados/management_token)
+    sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
+    database_pw=$(cat /var/lib/arvados/api_database_pw)
+    vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+
 cat >config/application.yml <<EOF
 $RAILS_ENV:
   uuid_prefix: $uuid_prefix

diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 7719fefce6c35912ffb63fc61152ae513d99a31f..801787e6a20e9bef6fb814c0fed675503fe72e89 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -12,14 +12,51 @@ fi
 
 . /usr/local/lib/arvbox/common.sh
 
+set -u
+
+if ! test -s /var/lib/arvados/api_uuid_prefix ; then
+  ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > /var/lib/arvados/api_uuid_prefix
+fi
 uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+
+if ! test -s /var/lib/arvados/api_secret_token ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/api_secret_token
+fi
 secret_token=$(cat /var/lib/arvados/api_secret_token)
+
+if ! test -s /var/lib/arvados/blob_signing_key ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/blob_signing_key
+fi
 blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+
+if ! test -s /var/lib/arvados/management_token ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/management_token
+fi
 management_token=$(cat /var/lib/arvados/management_token)
+
+if ! test -s /var/lib/arvados/sso_app_secret ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_app_secret
+fi
 sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
+
+if ! test -s /var/lib/arvados/vm-uuid ; then
+    echo $uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]') > /var/lib/arvados/vm-uuid
+fi
 vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+
+if ! test -f /var/lib/arvados/api_database_pw ; then
+    ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
+fi
 database_pw=$(cat /var/lib/arvados/api_database_pw)
 
+if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
+    psql postgres -c "create user arvados with password '$database_pw'"
+fi
+psql postgres -c "ALTER USER arvados WITH SUPERUSER;"
+
+if ! test -s /var/lib/arvados/workbench_secret_token ; then
+  ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/workbench_secret_token
+fi
 workbench_secret_key_base=$(cat /var/lib/arvados/workbench_secret_token)
 
 if test -s /var/lib/arvados/api_rails_env ; then

diff --git a/tools/arvbox/lib/arvbox/docker/service/certificate/run b/tools/arvbox/lib/arvbox/docker/service/certificate/run
index f951eef18d0e8e2690b4d4c64c446f2f52ae4ff2..6443b01793dd61aaa56e717aff71ec839a26c2c3 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/certificate/run
+++ b/tools/arvbox/lib/arvbox/docker/service/certificate/run
@@ -8,6 +8,8 @@ set -ex -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
+/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
 uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
 
 if ! openssl verify -CAfile $root_cert $root_cert ; then

diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/run b/tools/arvbox/lib/arvbox/docker/service/controller/run
index 56deed782e88da2c92c9d5fa46656135dd1e017b..7c16e08e2ebac969eb28bf5121d9e530069fca04 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/controller/run
+++ b/tools/arvbox/lib/arvbox/docker/service/controller/run
@@ -15,6 +15,6 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
 exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-controller

diff --git a/tools/arvbox/lib/arvbox/docker/service/sso/run-service b/tools/arvbox/lib/arvbox/docker/service/sso/run-service
index a7d3b1ca2eee874e09392cce97d521a47db39041..e30e34f7c116352e53e8d701d268115427f24b6c 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/sso/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sso/run-service
@@ -25,9 +25,6 @@ fi
 
 set -u
 
-if ! test -s /var/lib/arvados/api_uuid_prefix ; then
-  ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > /var/lib/arvados/api_uuid_prefix
-fi
 uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
 
 if ! test -s /var/lib/arvados/sso_secret_token ; then
@@ -62,9 +59,6 @@ sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config
 if ! test -f /var/lib/arvados/sso_database_setup ; then
    bundle exec rake db:setup
 
-   if ! test -s /var/lib/arvados/sso_app_secret ; then
-       ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_app_secret
-   fi
    app_secret=$(cat /var/lib/arvados/sso_app_secret)
 
    bundle exec rails console <<EOF

diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 9b139500b7fbe8f9f07be3b7538aafe49c768a91..06742cf82e204649336ff3d21ce676d0bb7a2dd4 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -35,9 +35,6 @@ fi
 
 set -u
 
-if ! test -s /var/lib/arvados/workbench_secret_token ; then
-  ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/workbench_secret_token
-fi
 secret_token=$(cat /var/lib/arvados/workbench_secret_token)
 
 if test -a /usr/src/arvados/apps/workbench/config/arvados_config.rb ; then