13647: Merge branch 'master' into 13647-keepstore-config

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/apps/workbench/test/integration_helper.rb b/apps/workbench/test/integration_helper.rb
index 0c66e59c8cce0d3e03ed90e9952be8ff264c647e..7209f2b6c968f0285d0a71d8df4a2c22777068ef 100644 (file)
--- a/apps/workbench/test/integration_helper.rb
+++ b/apps/workbench/test/integration_helper.rb
@@ -206,7 +206,7 @@ class ActionDispatch::IntegrationTest
   end
 
   teardown do
-    if not passed?
+    if !passed? && !skipped?
       screenshot
     end
     if Capybara.current_driver == :selenium

diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index 1d26deefa38bbe7c593d07d47dfe0d7d409cf49d..28a7fa51375bfd15acd9fc8b178eac7541376842 100644 (file)
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -348,3 +348,29 @@ end
 
 # Reset fixtures now (i.e., before any tests run).
 ActiveSupport::TestCase.reset_api_fixtures_now
+
+module Minitest
+  class Test
+    def capture_exceptions *args
+      begin
+        n = 0
+        begin
+          yield
+        rescue *PASSTHROUGH_EXCEPTIONS
+          raise
+        rescue Exception => e
+          n += 1
+          raise if n > 2 || e.is_a?(Skip)
+          STDERR.puts "Test failed, retrying (##{n})"
+          retry
+        end
+      rescue *PASSTHROUGH_EXCEPTIONS
+        raise
+      rescue Assertion => e
+        self.failures << e
+      rescue Exception => e
+        self.failures << UnexpectedError.new(e)
+      end
+    end
+  end
+end

diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 3d3ea6f0f82798953c2d3eac7a646fa1240735a9..7219edb37eb09a3bd89d41968a4f2f3516a5e51a 100644 (file)
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -18,6 +18,17 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
     /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
+# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
+RUN cd /usr/local/src \
+&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz \
+&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz.sig \
+&& tar xzf bash-4.4.12.tar.gz \
+&& cd bash-4.4.12 \
+&& ./configure --prefix=/usr/local/$( basename $( pwd ) ) \
+&& make \
+&& make install \
+&& ln -sf /usr/local/src/bash-4.4.12/bash /bin/bash
+
 # Install golang binary
 ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
 RUN ln -s /usr/local/go/bin/go /usr/local/bin/

diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile
index 0bfe80b70d87178619d0e9a8049cfa23f61a0acd..49d04aa7444a89f03cea6e838d2703eba5319ad3 100644 (file)
--- a/build/package-test-dockerfiles/centos7/Dockerfile
+++ b/build/package-test-dockerfiles/centos7/Dockerfile
@@ -18,6 +18,17 @@ RUN touch /var/lib/rpm/* && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3
 
+# Install Bash 4.4.12  // see https://dev.arvados.org/issues/15612
+RUN cd /usr/local/src \
+&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz \
+&& wget http://ftp.gnu.org/gnu/bash/bash-4.4.12.tar.gz.sig \
+&& tar xzf bash-4.4.12.tar.gz \
+&& cd bash-4.4.12 \
+&& ./configure --prefix=/usr/local/$( basename $( pwd ) ) \
+&& make \
+&& make install \
+&& ln -sf /usr/local/src/bash-4.4.12/bash /bin/bash
+
 # Add epel, we need it for the python-pam dependency
 RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 RUN rpm -ivh epel-release-latest-7.noarch.rpm

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 998b780b439edc924b4b41ae28548396eb46f04d..27228a37411c0d5cad2dbfb48ab322bcb4712c61 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1052,7 +1052,7 @@ test_govendor() {
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]}
 }
 
 test_sdk/ruby() {
@@ -1089,47 +1089,32 @@ test_services/nodemanager_integration() {
 
 test_apps/workbench_units() {
     local TASK="test:units"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
     local TASK="test:functionals"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
     local TASK="test:integration"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} 
 }
 
 test_apps/workbench_benchmark() {
     local TASK="test:benchmark"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]}
 }
 
 test_apps/workbench_profile() {
     local TASK="test:profile"
-    if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then
-        TASK="test"
-    fi
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]}
+        && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]}
 }
 
 install_deps() {

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index e56fbd489be3312c0e9159379d351a75acd20630..f474a6b23be33fdd58135396c1ca66e79611232e 100755 (executable)
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -242,6 +242,8 @@ run() {
         fi
         if ! test -d "$COMPOSER_ROOT" ; then
             git clone https://github.com/curoverse/composer.git "$COMPOSER_ROOT"
+            git -C "$COMPOSER_ROOT" checkout arvados-fork
+            git -C "$COMPOSER_ROOT" pull
         fi
         if ! test -d "$WORKBENCH2_ROOT" ; then
             git clone https://github.com/curoverse/arvados-workbench2.git "$WORKBENCH2_ROOT"

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index dbfa3f1124e167ccb4a5898897d36fb1bfa6534d..e6caa4624eaaabe82143856879f44a11d5f20adf 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -5,18 +5,22 @@
 FROM arvados/arvbox-base
 ARG arvados_version
 ARG sso_version=master
-ARG composer_version=master
+ARG composer_version=arvados-fork
 ARG workbench2_version=master
 
 RUN cd /usr/src && \
     git clone --no-checkout https://github.com/curoverse/arvados.git && \
     git -C arvados checkout ${arvados_version} && \
+    git -C arvados pull && \
     git clone --no-checkout https://github.com/curoverse/sso-devise-omniauth-provider.git sso && \
     git -C sso checkout ${sso_version} && \
+    git -C sso pull && \
     git clone --no-checkout https://github.com/curoverse/composer.git && \
     git -C composer checkout ${composer_version} && \
+    git -C composer pull && \
     git clone --no-checkout https://github.com/curoverse/arvados-workbench2.git workbench2 && \
-    git -C workbench2 checkout ${workbench2_version}
+    git -C workbench2 checkout ${workbench2_version} && \
+    git -C workbench2 pull
 
 ADD service/ /var/lib/arvbox/service
 RUN ln -sf /var/lib/arvbox/service /etc

diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 951b592ea69d1893b6c030f539ce8e8d8bcf925e..40034ee6e8e19c8bd1f97c9376e646fe21b05ea2 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -82,7 +82,7 @@ Clusters:
       Keepproxy:
         InternalURLs:
           "http://localhost:${services[keepproxy]}/": {}
-        ExternalURL: "http://$localip:${services[keepproxy-ssl]}/"
+        ExternalURL: "https://$localip:${services[keepproxy-ssl]}/"
       Websocket:
         ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
         InternalURLs:
@@ -104,7 +104,7 @@ Clusters:
         InternalURLs:
           "http://localhost:${services[keep-web]}/": {}
       Composer:
-        ExternalURL: "http://$localip:${services[composer]}"
+        ExternalURL: "https://$localip:${services[composer]}"
       Controller:
         ExternalURL: "https://$localip:${services[controller-ssl]}"
         InternalURLs:

diff --git a/tools/arvbox/lib/arvbox/docker/service/composer/run-service b/tools/arvbox/lib/arvbox/docker/service/composer/run-service
index fffa3daba825d226babdb91ba419b7a1723b0cab..f0733e0c2b86ded24445a968670169ca80c953eb 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/composer/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/composer/run-service
@@ -18,5 +18,7 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-echo "API_HOST: ${localip}:${services[controller-ssl]}" > /usr/src/composer/src/composer.yml
-exec node_modules/.bin/ng serve --host 0.0.0.0 --port 4200 --env=webdev
+cat >/usr/src/composer/src/composer.yml <<EOF
+API_HOST: ${localip}:${services[controller-ssl]}
+EOF
+exec node_modules/.bin/ng serve --host 0.0.0.0 --port 4200 --env=webdev --ssl=true --sslCert="${server_cert}" --sslKey="${server_cert_key}"

diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index d99d3c1cf8324c4dd33f9bcdd72a15fa4f015d7c..2c49dcae62f1a30be179735e15d42a6b1e148934 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -105,7 +105,7 @@ class Summarizer(object):
                                        self.label, uuid)
                         continue
                     logger.debug('%s: follow %s', self.label, uuid)
-                    child_summarizer = ProcessSummarizer(uuid)
+                    child_summarizer = NewSummarizer(uuid)
                     child_summarizer.stats_max = self.stats_max
                     child_summarizer.task_stats = self.task_stats
                     child_summarizer.tasks = self.tasks
@@ -526,7 +526,7 @@ def NewSummarizer(process_or_uuid, **kwargs):
 
 
 class ProcessSummarizer(Summarizer):
-    """Process is a job, pipeline, container, or container request."""
+    """Process is a job, pipeline, or container request."""
 
     def __init__(self, process, label=None, **kwargs):
         rdr = None
@@ -535,17 +535,16 @@ class ProcessSummarizer(Summarizer):
             label = self.process.get('name', self.process['uuid'])
         # Pre-Arvados v1.4 everything is in 'log'
         # For 1.4+ containers have no logs and container_requests have them in 'log_uuid', not 'log'
-        log_collection = self.process.get('log')
-        if not log_collection:
-            log_collection = self.process.get('log_uuid')
-        if log_collection:
+        log_collection = self.process.get('log', self.process.get('log_uuid'))
+        if log_collection and self.process.get('state') != 'Uncommitted': # arvados.util.CR_UNCOMMITTED:
             try:
                 rdr = crunchstat_summary.reader.CollectionReader(log_collection)
             except arvados.errors.NotFoundError as e:
                 logger.warning("Trying event logs after failing to read "
                                "log collection %s: %s", self.process['log'], e)
         if rdr is None:
-            rdr = crunchstat_summary.reader.LiveLogReader(self.process['uuid'])
+            uuid = self.process.get('container_uuid', self.process.get('uuid'))
+            rdr = crunchstat_summary.reader.LiveLogReader(uuid)
             label = label + ' (partial)'
         super(ProcessSummarizer, self).__init__(rdr, label=label, **kwargs)
         self.existing_constraints = self.process.get('runtime_constraints', {})