build/test-library.sh

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 COLUMNS=80
   6 . `dirname "$(readlink -f "$0")"`/run-library.sh
   7
   8 declare -a failures
   9
  10 clear_temp() {
  11     if [[ -z "$temp" ]]; then
  12         # we didn't even get as far as making a temp dir
  13         :
  14     elif [[ -z "$temp_preserve" ]]; then
  15         rm -rf "$temp"
  16     else
  17         echo "Leaving behind temp dirs in $temp"
  18     fi
  19 }
  20
  21 fatal() {
  22     clear_temp
  23     echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
  24     exit 1
  25 }
  26
  27 rotate_logfile() {
  28   # i.e.  rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
  29   # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
  30   if [[ -f "$1/$2" ]]; then
  31     THEDATE=`date +%Y%m%d%H%M%S`
  32     mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
  33     gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
  34   fi
  35 }
  36
  37 exit_cleanly() {
  38     trap - INT
  39     #create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
  40     rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
  41     stop_services
  42     rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
  43     report_outcomes
  44     clear_temp
  45     exit ${#failures}
  46 }
  47
  48 retry() {
  49     local remain="${repeat}"
  50     while :
  51     do
  52         if ${@}; then
  53             if [[ "$remain" -gt 1 ]]; then
  54                 remain=$((${remain}-1))
  55                 title "Repeating ${remain} more times"
  56             else
  57                 break
  58             fi
  59         elif [[ "$retry" == 1 ]]; then
  60             read -p 'Try again? [Y/n] ' x
  61             if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
  62             then
  63                 break
  64             fi
  65         else
  66             break
  67         fi
  68     done
  69 }
  70
  71 bundle_install_trylocal() {
  72     (
  73         set -e
  74         echo "(Running bundle install --local. 'could not find package' messages are OK.)"
  75         if ! bundle install --local --no-deployment; then
  76             echo "(Running bundle install again, without --local.)"
  77             bundle install --no-deployment
  78         fi
  79         bundle package --all
  80     )
  81 }
  82
  83 declare -A did_install
  84
  85 should_install() {
  86     if [[ -n "${did_install[$1]}" ]] ; then
  87         return 1
  88     fi
  89
  90     if [[ -n "${include_install[@]}" && -z "${include_install[$1]}" ]] ; then
  91         title "Skipping $1 install because not in include_install"
  92         did_install[$1]=1
  93         return 1
  94     fi
  95
  96     if [[ -n "${exclude_install[$1]}" ]] ; then
  97         title "Skipping $1 install because of exclude_install"
  98         did_install[$1]=1
  99         return 1
 100     fi
 101
 102     return 0
 103 }
 104
 105 do_install() {
 106     if [[ -z "${did_install[$1]}" ]] ; then
 107         retry do_install_once ${@}
 108     fi
 109 }
 110
 111 do_install_once() {
 112     local name="$1"
 113     title "Running $name install"
 114     timer_reset
 115     if [[ "$2" == "go" ]]
 116     then
 117         do_install gopath
 118         go get -t -v "git.curoverse.com/arvados.git/$name"
 119     elif [[ "$2" == "pip" ]]
 120     then
 121         if ! should_install "$1" ; then
 122             return
 123         fi
 124
 125         # $3 can name a path directory for us to use, including trailing
 126         # slash; e.g., the bin/ subdirectory of a virtualenv.
 127
 128         # Need to change to a different directory after creating
 129         # the source dist package to avoid a pip bug.
 130         # see https://arvados.org/issues/5766 for details.
 131
 132         # Also need to install twice, because if it believes the package is
 133         # already installed, pip it won't install it.  So the first "pip
 134         # install" ensures that the dependencies are met, the second "pip
 135         # install" ensures that we've actually installed the local package
 136         # we just built.
 137         cd "$WORKSPACE/$name" \
 138             && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
 139             && cd "$WORKSPACE" \
 140             && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$name/dist"/*.tar.gz \
 141             && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$name/dist"/*.tar.gz
 142     else
 143         shift
 144         "install_$name" "$@"
 145     fi
 146     result=$?
 147     checkexit $result "$name install"
 148     did_install[$name]=1
 149     title "End of $name install (`timer`)"
 150     return $result
 151 }
 152
 153 declare -A testargs
 154
 155 do_test_once() {
 156     local result
 157
 158     if [[ "$2" == "go" ]]
 159     then
 160         do_install gopath
 161     fi
 162
 163     title "Running $1 tests"
 164     timer_reset
 165     if [[ "$2" == "go" ]]
 166     then
 167         covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
 168         coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
 169         # We do "go get -t" here to catch compilation errors
 170         # before trying "go test". Otherwise, coverage-reporting
 171         # mode makes Go show the wrong line numbers when reporting
 172         # compilation errors.
 173         go get -t "git.curoverse.com/arvados.git/$1" && \
 174             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
 175             [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
 176             if [[ -n "${testargs[$1]}" ]]
 177         then
 178             # "go test -check.vv giturl" doesn't work, but this
 179             # does:
 180             go test ${short:+-short} ${testargs[$1]}
 181         else
 182             # The above form gets verbose even when testargs is
 183             # empty, so use this form in such cases:
 184             go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
 185         fi
 186         result=${result:-$?}
 187         if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
 188         then
 189             go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
 190             rm "$WORKSPACE/tmp/.$covername.tmp"
 191         fi
 192     elif [[ "$2" == "pip" ]]
 193     then
 194         tries=0
 195         cd "$WORKSPACE/$1" && while :
 196         do
 197             tries=$((${tries}+1))
 198             # $3 can name a path directory for us to use, including trailing
 199             # slash; e.g., the bin/ subdirectory of a virtualenv.
 200             "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
 201             result=$?
 202             if [[ ${tries} < 3 && ${result} == 137 ]]
 203             then
 204                 printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1"
 205                 continue
 206             else
 207                 break
 208             fi
 209         done
 210     elif [[ "$2" != "" ]]
 211     then
 212         "test_$2"
 213     else
 214         "test_$1"
 215     fi
 216     result=${result:-$?}
 217     checkexit $result "$1 tests"
 218     title "End of $1 tests (`timer`)"
 219     return $result
 220 }
 221
 222 do_test() {
 223     retry do_test_once ${@}
 224 }
 225
 226 mktmpdir() {
 227     if [[ -z "$temp" ]]; then
 228         temp="$(mktemp -d)"
 229     fi
 230
 231     if ! [[ -d "$temp" ]] ; then
 232         mkdir -p "$temp"
 233     fi
 234
 235     tmpdir="${1}"
 236     if [[ -z "${!tmpdir}" ]]; then
 237         eval "$tmpdir"="$temp/$tmpdir"
 238     fi
 239     if ! [[ -d "${!tmpdir}" ]]; then
 240         mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)"
 241     fi
 242 }
 243
 244 install_gopath() {
 245     mktmpdir GOPATH
 246     export GOPATH
 247
 248     if ! should_install gopath ; then
 249         return
 250     fi
 251
 252     mkdir -p "$GOPATH/src/git.curoverse.com"
 253     rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
 254     for d in \
 255         "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
 256             "$GOPATH/src/git.curoverse.com/arvados.git"; do
 257         [[ -d "$d" ]] && rmdir "$d"
 258         [[ -h "$d" ]] && rm "$d"
 259     done
 260     ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
 261         || fatal "symlink failed"
 262     go get -v github.com/kardianos/govendor \
 263         || fatal "govendor install failed"
 264     cd "$GOPATH/src/git.curoverse.com/arvados.git" \
 265         || fatal
 266     # Remove cached source dirs in workdir. Otherwise, they won't qualify
 267     # as +missing or +external below, and we won't be able to detect that
 268     # they're missing from vendor/vendor.json.
 269     rm -rf vendor/*/
 270     go get -v -d ...
 271     "$GOPATH/bin/govendor" sync \
 272         || fatal "govendor sync failed"
 273     [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
 274         || fatal "vendor/vendor.json has unused or missing dependencies -- try:
 275 * govendor remove +unused
 276 * govendor add +missing +external
 277 "
 278 }
 279
 280 install_virtualenv() {
 281     local venvdest="$1"; shift
 282     if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
 283         virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
 284     fi
 285     if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then
 286         # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy
 287         "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8'
 288     else
 289         "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
 290     fi
 291     # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
 292     "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
 293 }
 294
 295
 296 install_py2_virtualenv() {
 297     mktmpdir VENVDIR
 298
 299     if ! should_install py2_virtualenv ; then
 300         return
 301     fi
 302
 303     do_install virtualenv "$VENVDIR" --python python2.7
 304     . "$VENVDIR/bin/activate"
 305
 306     # Needed for run_test_server.py which is used by certain (non-Python) tests.
 307     pip freeze 2>/dev/null | egrep ^PyYAML= \
 308         || pip install --no-cache-dir PyYAML >/dev/null \
 309         || fatal "pip install PyYAML failed"
 310
 311     # Preinstall libcloud, because nodemanager "pip install"
 312     # won't pick it up by default.
 313     pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
 314         || pip install --pre --ignore-installed --no-cache-dir "apache-libcloud>=$LIBCLOUD_PIN" >/dev/null \
 315         || fatal "pip install apache-libcloud failed"
 316
 317     # We need an unreleased (as of 2017-08-17) llfuse bugfix, otherwise our fuse test suite deadlocks.
 318     pip freeze | grep -x llfuse==1.2.0 || (
 319         set -e
 320         yes | pip uninstall llfuse || true
 321         cython --version || fatal "no cython; try sudo apt-get install cython"
 322         cd "$temp"
 323         (cd python-llfuse 2>/dev/null || git clone https://github.com/curoverse/python-llfuse)
 324         cd python-llfuse
 325         git checkout 620722fd990ea642ddb8e7412676af482c090c0c
 326         git checkout setup.py
 327         sed -i -e "s:'1\\.2':'1.2.0':" setup.py
 328         python setup.py build_cython
 329         python setup.py install --force
 330     ) || fatal "llfuse fork failed"
 331     pip freeze | grep -x llfuse==1.2.0 || fatal "error: installed llfuse 1.2.0 but '$(pip freeze | grep llfuse)' ???"
 332
 333     deactivate
 334 }
 335
 336 install_py3_virtualenv() {
 337     mktmpdir VENVDIR3
 338     do_install virtualenv "$VENVDIR3" --python python3
 339 }
 340
 341 install_apiserver() {
 342     if ! should_install apiserver ; then
 343         return
 344     fi
 345
 346     cd "$WORKSPACE/services/api" \
 347         && RAILS_ENV=test bundle_install_trylocal
 348
 349     rm -f config/environments/test.rb
 350     cp config/environments/test.rb.example config/environments/test.rb
 351
 352     if [ -n "$CONFIGSRC" ]
 353     then
 354         for f in database.yml
 355         do
 356             cp "$CONFIGSRC/$f" config/ || fatal "$f"
 357         done
 358     fi
 359
 360     # Clear out any lingering postgresql connections to the test
 361     # database, so that we can drop it. This assumes the current user
 362     # is a postgresql superuser.
 363     cd "$WORKSPACE/services/api" \
 364         && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
 365         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 366
 367     mkdir -p "$WORKSPACE/services/api/tmp/pids"
 368
 369     cert="$WORKSPACE/services/api/tmp/self-signed"
 370     if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then
 371         (
 372             dir="$WORKSPACE/services/api/tmp"
 373             set -ex
 374             openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" </dev/null
 375             openssl x509 -req -in "$cert.csr" -signkey "$cert.key" -out "$cert.pem" -days 3650 -extfile <(printf 'subjectAltName=DNS:localhost,DNS:::1,DNS:0.0.0.0,DNS:127.0.0.1,IP:::1,IP:0.0.0.0,IP:127.0.0.1')
 376         ) || return 1
 377     fi
 378
 379     cd "$WORKSPACE/services/api" \
 380         && rm -rf tmp/git \
 381         && mkdir -p tmp/git \
 382         && cd tmp/git \
 383         && tar xf ../../test/test.git.tar \
 384         && mkdir -p internal.git \
 385         && git --git-dir internal.git init \
 386             || return 1
 387
 388     cd "$WORKSPACE/services/api" \
 389         && RAILS_ENV=test bundle exec rake db:drop \
 390         && RAILS_ENV=test bundle exec rake db:setup \
 391         && RAILS_ENV=test bundle exec rake db:fixtures:load
 392 }
 393
 394
 395 start_services() {
 396     if [[ -n "${did_install[start_services]}" ]] ; then
 397        return
 398     fi
 399
 400     do_install py2_virtualenv
 401
 402     # Install the Python SDK early. Various other test suites (like
 403     # keepproxy) bring up run_test_server.py, which imports the arvados
 404     # module. We can't actually *test* the Python SDK yet though, because
 405     # its own test suite brings up some of those other programs (like
 406     # keepproxy).
 407     . "$VENVDIR/bin/activate"
 408     do_install sdk/python pip
 409
 410     do_install apiserver
 411     do_install services/keepstore go
 412     do_install services/keepproxy go
 413     do_install services/keep-web go
 414     do_install services/ws go
 415     do_install services/arv-git-httpd go
 416
 417     echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
 418     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
 419         mkdir -p "$WORKSPACE/services/api/log"
 420     fi
 421     # Remove empty api.pid file if it exists
 422     if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
 423         rm -f "$WORKSPACE/tmp/api.pid"
 424     fi
 425     cd "$WORKSPACE" \
 426         && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
 427         && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
 428         && export ARVADOS_TEST_API_INSTALLED="$$" \
 429         && python sdk/python/tests/run_test_server.py start_keep_proxy \
 430         && python sdk/python/tests/run_test_server.py start_keep-web \
 431         && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
 432         && python sdk/python/tests/run_test_server.py start_ws \
 433         && python sdk/python/tests/run_test_server.py start_nginx \
 434         && (env | egrep ^ARVADOS)
 435     did_install[start_services]=1
 436 }
 437
 438 stop_services() {
 439     if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then
 440         return
 441     fi
 442     unset ARVADOS_TEST_API_HOST
 443     cd "$WORKSPACE" \
 444         && python sdk/python/tests/run_test_server.py stop_nginx \
 445         && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
 446         && python sdk/python/tests/run_test_server.py stop_ws \
 447         && python sdk/python/tests/run_test_server.py stop_keep-web \
 448         && python sdk/python/tests/run_test_server.py stop_keep_proxy \
 449         && python sdk/python/tests/run_test_server.py stop
 450     did_install[start_services]=""
 451 }
 452
 453 install_run_test_server() {
 454     start_services
 455 }