--- /dev/null
+export WORKSPACE?=$(shell pwd)
+help:
+ @echo >&2
+ @echo >&2 "There is no default make target here. Did you mean 'make test'?"
+ @echo >&2
+ @echo >&2 "More info:"
+ @echo >&2 " Installing --> http://doc.arvados.org/install"
+ @echo >&2 " Developing/contributing --> https://dev.arvados.org"
+ @echo >&2 " Project home --> https://arvados.org"
+ @echo >&2
+ @false
+test:
+ build/run-tests.sh ${TEST_FLAGS}
+packages:
+ build/run-build-packages-all-targets.sh ${PACKAGES_FLAGS}
+test-packages:
+ build/run-build-packages-all-targets.sh --test-packages ${PACKAGES_FLAGS}
# FIXME: Remove this line after #6885 is done.
fpm_args+=(--iteration 2)
-
-# FIXME: Remove once support for llfuse 0.42+ is in place
-fpm_args+=(-v 0.41.1)
--- /dev/null
+#!/bin/bash
+
+build=$1
+file=$2
+outputdir=$3
+
+usage() {
+ echo "./$0 build_number file_to_parse output_dir"
+ echo "this script will use the build output to generate *csv and *txt"
+ echo "for jenkins plugin plot https://github.com/jenkinsci/plot-plugin/"
+}
+
+if [ $# -ne 3 ]
+then
+ usage
+ exit 1
+fi
+
+if [ ! -e $file ]
+then
+ usage
+ echo "$file doesn't exist! exiting"
+ exit 2
+fi
+if [ ! -w $outputdir ]
+then
+ usage
+ echo "$outputdir isn't writeable! exiting"
+ exit 3
+fi
+
+#------------------------------
+## MAXLINE is the amount of lines that will read after the pattern
+## is match (the logfile could be hundred thousands lines long).
+## 1000 should be safe enough to capture all the output of the individual test
+MAXLINES=1000
+
+## TODO: check $build and $file make sense
+
+for test in \
+ test_Create_and_show_large_collection_with_manifest_text_of_20000000 \
+ test_Create,_show,_and_update_description_for_large_collection_with_manifest_text_of_100000 \
+ test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
+do
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ (zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail -n +1 | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$cleaned_test-$build.txt
+ result=$?
+ if [ $result -eq 0 ]
+ then
+ echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
+ #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
+ else
+ echo "$test was't found on $file"
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ > $outputdir/$cleaned_test.csv
+ fi
+done
--- /dev/null
+LIBCLOUD_PIN=0.20.2.dev1
\ No newline at end of file
--- /dev/null
+*/generated
+common-generated/
--- /dev/null
+all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
+
+centos6/generated: common-generated-all
+ test -d centos6/generated || mkdir centos6/generated
+ cp -rlt centos6/generated common-generated/*
+
+debian7/generated: common-generated-all
+ test -d debian7/generated || mkdir debian7/generated
+ cp -rlt debian7/generated common-generated/*
+
+debian8/generated: common-generated-all
+ test -d debian8/generated || mkdir debian8/generated
+ cp -rlt debian8/generated common-generated/*
+
+ubuntu1204/generated: common-generated-all
+ test -d ubuntu1204/generated || mkdir ubuntu1204/generated
+ cp -rlt ubuntu1204/generated common-generated/*
+
+ubuntu1404/generated: common-generated-all
+ test -d ubuntu1404/generated || mkdir ubuntu1404/generated
+ cp -rlt ubuntu1404/generated common-generated/*
+
+common-generated-all: common-generated/golang-amd64.tar.gz
+
+common-generated/golang-amd64.tar.gz: common-generated
+ wget -cqO common-generated/golang-amd64.tar.gz http://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
+
+common-generated:
+ mkdir common-generated
--- /dev/null
+==================
+DOCKER IMAGE BUILD
+==================
+
+1. `make`
+2. `cd DISTRO`
+3. `docker build -t arvados/build:DISTRO .`
+
+==============
+BUILD PACKAGES
+==============
+
+`docker run -v /path/to/your/arvados-dev/jenkins:/jenkins -v /path/to/your/arvados:/arvados arvados/build:DISTRO`
--- /dev/null
+#!/bin/bash
+
+make
+
+for target in `find -maxdepth 1 -type d |grep -v generated`; do
+ if [[ "$target" == "." ]]; then
+ continue
+ fi
+ target=${target#./}
+ echo $target
+ cd $target
+ docker build -t arvados/build:$target .
+ cd ..
+done
+
+
--- /dev/null
+FROM centos:6
+MAINTAINER Brett Smith <brett@curoverse.com>
+
+# Install build dependencies provided in base distribution
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar scl-utils centos-release-SCL postgresql-devel
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python27 python33
+RUN scl enable python33 "easy_install-3.3 pip" && scl enable python27 "easy_install-2.7 pip"
+
+RUN cd /tmp && \
+ curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
+ rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
+ sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
+
+RUN touch /var/lib/rpm/* && yum install --assumeyes git
+
+ENV WORKSPACE /arvados
+CMD ["scl", "enable", "python33", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]
--- /dev/null
+FROM debian:wheezy
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian7"]
--- /dev/null
+FROM debian:jessie
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
--- /dev/null
+FROM ubuntu:precise
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1204"]
--- /dev/null
+FROM ubuntu:trusty
+MAINTAINER Brett Smith <brett@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
--- /dev/null
+FROM centos:6
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN yum -q install --assumeyes scl-utils centos-release-SCL \
+ which tar
+
+# Install RVM
+RUN touch /var/lib/rpm/* && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundle fpm
+
+RUN cd /tmp && \
+ curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
+ rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
+ sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file
--- /dev/null
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos6
+gpgcheck=0
+enabled=1
--- /dev/null
+FROM debian:7
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl procps && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list
--- /dev/null
+FROM debian:8
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
--- /dev/null
+FROM ubuntu:precise
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file
--- /dev/null
+FROM ubuntu:trusty
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+
+set -eu
+
+FAIL=0
+
+echo
+
+while read so && [ -n "$so" ]; do
+ if ldd "$so" | grep "not found" ; then
+ echo "^^^ Missing while scanning $so ^^^"
+ FAIL=1
+ fi
+done <<EOF
+$(find -name '*.so')
+EOF
+
+if test -x "/jenkins/package-testing/test-package-$1.sh" ; then
+ if ! "/jenkins/package-testing/test-package-$1.sh" ; then
+ FAIL=1
+ fi
+fi
+
+if test $FAIL = 0 ; then
+ echo "Package $1 passed"
+fi
+
+exit $FAIL
--- /dev/null
+#!/bin/bash
+
+set -eu
+
+# Multiple .deb based distros symlink to this script, so extract the target
+# from the invocation path.
+target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
+
+apt-get -qq update
+apt-get --assume-yes --force-yes install "$1"
+
+dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
+
+set +e
+diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" > "$ARV_PACKAGES_DIR/$1.diff"
+set -e
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+dpkg-deb -x $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) .
+
+while read so && [ -n "$so" ]; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+done <<EOF
+$(find -name '*.so')
+EOF
+
+exec /jenkins/package-testing/common-test-packages.sh "$1"
--- /dev/null
+#!/bin/sh
+set -e
+cd /var/www/arvados-api/current/
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ apt-get install -y nginx
+ dpkg-reconfigure arvados-api-server
+ ;;
+ centos6)
+ yum install --assumeyes httpd
+ yum reinstall --assumeyes arvados-api-server
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-api-server.gems"
--- /dev/null
+#!/bin/sh
+exec python <<EOF
+import libcloud.compute.types
+import libcloud.compute.providers
+libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
+print "Successfully imported compatible libcloud library"
+EOF
--- /dev/null
+#!/bin/bash
+
+set -e
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ FORMAT=deb
+ ;;
+ centos6)
+ FORMAT=rpm
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+title () {
+ txt="********** $1 **********"
+ printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
+}
+
+checkexit() {
+ if [[ "$1" != "0" ]]; then
+ title "!!!!!! $2 FAILED !!!!!!"
+ fi
+}
+
+
+# Find the SSO server package
+
+cd "$WORKSPACE"
+
+if [[ ! -d "/var/www/arvados-sso" ]]; then
+ echo "/var/www/arvados-sso should exist"
+ exit 1
+fi
+
+if [[ ! -e "/etc/arvados/sso/application.yml" ]]; then
+ mkdir -p /etc/arvados/sso/
+ RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
+ cp config/application.yml.example /etc/arvados/sso/application.yml
+ sed -i -e 's/uuid_prefix: ~/uuid_prefix: zzzzz/' /etc/arvados/sso/application.yml
+ sed -i -e "s/secret_token: ~/secret_token: $RANDOM_PASSWORD/" /etc/arvados/sso/application.yml
+fi
+
+if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then
+ # We need to set up our database configuration now.
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # postgres packaging on CentOS6 is kind of primitive, needs an initdb
+ service postgresql initdb
+ if [ "$TARGET" = "centos6" ]; then
+ sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
+ sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
+ fi
+ fi
+ service postgresql start
+
+ RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
+ cat >/etc/arvados/sso/database.yml <<EOF
+production:
+ adapter: postgresql
+ encoding: utf8
+ database: sso_provider_production
+ username: sso_provider_user
+ password: $RANDOM_PASSWORD
+ host: localhost
+EOF
+
+ su postgres -c "psql -c \"CREATE USER sso_provider_user WITH PASSWORD '$RANDOM_PASSWORD'\""
+ su postgres -c "createdb sso_provider_production -O sso_provider_user"
+fi
+
+if [[ "$FORMAT" == "deb" ]]; then
+ # Test 2: the package should reconfigure cleanly
+ dpkg-reconfigure arvados-sso-server || EXITCODE=3
+
+ cd /var/www/arvados-sso/current/
+ /usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-sso-server.gems"
+
+ # Test 3: the package should remove cleanly
+ apt-get remove arvados-sso-server --yes || EXITCODE=3
+
+ checkexit $EXITCODE "apt-get remove arvados-sso-server --yes"
+
+ # Test 4: the package configuration should remove cleanly
+ dpkg --purge arvados-sso-server || EXITCODE=4
+
+ checkexit $EXITCODE "dpkg --purge arvados-sso-server"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=4
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+ # Test 5: the package should remove cleanly with --purge
+
+ apt-get remove arvados-sso-server --purge --yes || EXITCODE=5
+
+ checkexit $EXITCODE "apt-get remove arvados-sso-server --purge --yes"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=5
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+elif [[ "$FORMAT" == "rpm" ]]; then
+
+ # Set up Nginx first
+ # (courtesy of https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/el6/install_passenger.html)
+ yum install -q -y epel-release pygpgme curl
+ curl --fail -sSLo /etc/yum.repos.d/passenger.repo https://oss-binaries.phusionpassenger.com/yum/definitions/el-passenger.repo
+ yum install -q -y nginx passenger
+ sed -i -e 's/^# passenger/passenger/' /etc/nginx/conf.d/passenger.conf
+ # Done setting up Nginx
+
+ # Test 2: the package should reinstall cleanly
+ yum --assumeyes reinstall arvados-sso-server || EXITCODE=3
+
+ cd /var/www/arvados-sso/current/
+ /usr/local/rvm/bin/rvm-exec default bundle list >$ARV_PACKAGES_DIR/arvados-sso-server.gems
+
+ # Test 3: the package should remove cleanly
+ yum -q -y remove arvados-sso-server || EXITCODE=3
+
+ checkexit $EXITCODE "yum -q -y remove arvados-sso-server"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=3
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+fi
+
+if [[ "$EXITCODE" == "0" ]]; then
+ echo "Testing complete, no errors!"
+else
+ echo "Errors while testing!"
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/sh
+set -e
+cd /var/www/arvados-workbench/current/
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ apt-get install -y nginx
+ dpkg-reconfigure arvados-workbench
+ ;;
+ centos6)
+ yum install --assumeyes httpd
+ yum reinstall --assumeyes arvados-workbench
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-workbench.gems"
--- /dev/null
+#!/bin/sh
+
+exec python <<EOF
+import arvados_fuse
+print "Successfully imported arvados_fuse"
+EOF
--- /dev/null
+#!/bin/sh
+
+exec python <<EOF
+import arvados
+print "Successfully imported arvados"
+EOF
--- /dev/null
+#!/bin/bash
+
+set -eu
+
+yum -q clean all
+touch /var/lib/rpm/*
+
+export ARV_PACKAGES_DIR=/arvados/packages/centos6
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
+
+yum install --assumeyes $1
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
+
+set +e
+diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff"
+set -e
+
+SCL=""
+if scl enable python27 true 2>/dev/null ; then
+ SCL="scl enable python27"
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
+
+shared=$(find -name '*.so')
+if test -n "$shared" ; then
+ for so in $shared ; do
+ echo
+ echo "== Packages dependencies for $so =="
+ $SCL ldd "$so" \
+ | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+ done
+fi
+
+if test -n "$SCL" ; then
+ exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'"
+else
+ exec /jenkins/package-testing/common-test-packages.sh "$1"
+fi
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+When run-build-packages.sh builds a Rails package, it generates the package's pre/post-inst/rm scripts by concatenating:
+
+1. package_name.sh, which defines variables about where package files live and some human-readable names about them.
+2. step2.sh, which uses those to define some utility variables and set defaults for things that aren't set.
+3. stepname.sh, like postinst.sh, prerm.sh, etc., which uses all this information to do the actual work.
+
+Since our build process is a tower of shell scripts, concatenating files seemed like the least worst option to share code between these files and packages. More advanced code generation would've been too much trouble to integrate into our build process at this time. Trying to inject portions of files into other files seemed error-prone and likely to introduce bugs to the end result.
+
+postinst.sh lets the early parts define a few hooks to control behavior:
+
+* After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles. By default this is a noop function (in step2.sh). API server defines this to set up the old omniauth.rb conffile.
+* Before it restarts nginx, it calls setup_before_nginx_restart. By default this is a noop function (in step2.sh). API server defines this to set up the internal git repository, if necessary.
+* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. SSO server uses db:schema:load. Workbench doesn't set this, which causes the postinst to skip all database work.
+* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task. SSO clears this flag (it doesn't have that task code).
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-api-server
+INSTALL_PATH=/var/www/arvados-api
+CONFIG_PATH=/etc/arvados/api
+DOC_URL="http://doc.arvados.org/install/install-api-server.html#configure"
+
+RAILSPKG_DATABASE_LOAD_TASK=db:structure:load
+setup_extra_conffiles() {
+ setup_conffile initializers/omniauth.rb
+}
+
+setup_before_nginx_restart() {
+ # initialize git_internal_dir
+ # usually /var/lib/arvados/internal.git (set in application.default.yml )
+ if [ "$APPLICATION_READY" = "1" ]; then
+ GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:check 2>&1 | grep git_internal_dir | awk '{ print $2 }')
+ if [ ! -e "$GIT_INTERNAL_DIR" ]; then
+ run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
+ mkdir -p "$GIT_INTERNAL_DIR"
+ run_and_report "Initializing git_internal_dir '$GIT_INTERNAL_DIR'" \
+ git init --quiet --bare $GIT_INTERNAL_DIR
+ else
+ echo "Initializing git_internal_dir $GIT_INTERNAL_DIR: directory exists, skipped."
+ fi
+ run_and_report "Making sure '$GIT_INTERNAL_DIR' has the right permission" \
+ chown -R "$WWW_OWNER:" "$GIT_INTERNAL_DIR"
+ else
+ echo "Initializing git_internal_dir... skipped."
+ fi
+}
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-sso-server
+INSTALL_PATH=/var/www/arvados-sso
+CONFIG_PATH=/etc/arvados/sso
+DOC_URL="http://doc.arvados.org/install/install-sso.html#configure"
+RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
+RAILSPKG_SUPPORTS_CONFIG_CHECK=0
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-workbench
+INSTALL_PATH=/var/www/arvados-workbench
+CONFIG_PATH=/etc/arvados/workbench
+DOC_URL="http://doc.arvados.org/install/install-workbench-app.html#configure"
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+set -e
+
+DATABASE_READY=1
+APPLICATION_READY=1
+
+if [ -s "$HOME/.rvm/scripts/rvm" ] || [ -s "/usr/local/rvm/scripts/rvm" ]; then
+ COMMAND_PREFIX="/usr/local/rvm/bin/rvm-exec default"
+else
+ COMMAND_PREFIX=
+fi
+
+report_not_ready() {
+ local ready_flag="$1"; shift
+ local config_file="$1"; shift
+ if [ "1" != "$ready_flag" ]; then cat >&2 <<EOF
+
+PLEASE NOTE:
+
+The $PACKAGE_NAME package was not configured completely because
+$config_file needs some tweaking.
+Please refer to the documentation at
+<$DOC_URL> for more details.
+
+When $(basename "$config_file") has been modified,
+reconfigure or reinstall this package.
+
+EOF
+ fi
+}
+
+report_web_service_warning() {
+ local warning="$1"; shift
+ cat >&2 <<EOF
+
+WARNING: $warning.
+
+To override, set the WEB_SERVICE environment variable to the name of the service
+hosting the Rails server.
+
+For Debian-based systems, then reconfigure this package with dpkg-reconfigure.
+
+For RPM-based systems, then reinstall this package.
+
+EOF
+}
+
+run_and_report() {
+ # Usage: run_and_report ACTION_MSG CMD
+ # This is the usual wrapper that prints ACTION_MSG, runs CMD, then writes
+ # a message about whether CMD succeeded or failed. Returns the exit code
+ # of CMD.
+ local action_message="$1"; shift
+ local retcode=0
+ echo -n "$action_message..."
+ if "$@"; then
+ echo " done."
+ else
+ retcode=$?
+ echo " failed."
+ fi
+ return $retcode
+}
+
+setup_confdirs() {
+ for confdir in "$@"; do
+ if [ ! -d "$confdir" ]; then
+ install -d -g "$WWW_OWNER" -m 0750 "$confdir"
+ fi
+ done
+}
+
+setup_conffile() {
+ # Usage: setup_conffile CONFFILE_PATH [SOURCE_PATH]
+ # Both paths are relative to RELEASE_CONFIG_PATH.
+ # This function will try to safely ensure that a symbolic link for
+ # the configuration file points from RELEASE_CONFIG_PATH to CONFIG_PATH.
+ # If SOURCE_PATH is given, this function will try to install that file as
+ # the configuration file in CONFIG_PATH, and return 1 if the file in
+ # CONFIG_PATH is unmodified from the source.
+ local conffile_relpath="$1"; shift
+ local conffile_source="$1"
+ local release_conffile="$RELEASE_CONFIG_PATH/$conffile_relpath"
+ local etc_conffile="$CONFIG_PATH/$(basename "$conffile_relpath")"
+
+ # Note that -h can return true and -e will return false simultaneously
+ # when the target is a dangling symlink. We're okay with that outcome,
+ # so check -h first.
+ if [ ! -h "$release_conffile" ]; then
+ if [ ! -e "$release_conffile" ]; then
+ ln -s "$etc_conffile" "$release_conffile"
+ # If there's a config file in /var/www identical to the one in /etc,
+ # overwrite it with a symlink after porting its permissions.
+ elif cmp --quiet "$release_conffile" "$etc_conffile"; then
+ local ownership="$(stat -c "%u:%g" "$release_conffile")"
+ local owning_group="${ownership#*:}"
+ if [ 0 != "$owning_group" ]; then
+ chgrp "$owning_group" "$CONFIG_PATH" /etc/arvados
+ fi
+ chown "$ownership" "$etc_conffile"
+ chmod --reference="$release_conffile" "$etc_conffile"
+ ln --force -s "$etc_conffile" "$release_conffile"
+ fi
+ fi
+
+ if [ -n "$conffile_source" ]; then
+ if [ ! -e "$etc_conffile" ]; then
+ install -g "$WWW_OWNER" -m 0640 \
+ "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"
+ return 1
+ # Even if $etc_conffile already existed, it might be unmodified from
+ # the source. This is especially likely when a user installs, updates
+ # database.yml, then reconfigures before they update application.yml.
+ # Use cmp to be sure whether $etc_conffile is modified.
+ elif cmp --quiet "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"; then
+ return 1
+ fi
+ fi
+}
+
+prepare_database() {
+ DB_MIGRATE_STATUS=`$COMMAND_PREFIX bundle exec rake db:migrate:status 2>&1 || true`
+ if echo $DB_MIGRATE_STATUS | grep -qF 'Schema migrations table does not exist yet.'; then
+ # The database exists, but the migrations table doesn't.
+ run_and_report "Setting up database" $COMMAND_PREFIX bundle exec \
+ rake "$RAILSPKG_DATABASE_LOAD_TASK" db:seed
+ elif echo $DB_MIGRATE_STATUS | grep -q '^database: '; then
+ run_and_report "Running db:migrate" \
+ $COMMAND_PREFIX bundle exec rake db:migrate
+ elif echo $DB_MIGRATE_STATUS | grep -q 'database .* does not exist'; then
+ if ! run_and_report "Running db:setup" \
+ $COMMAND_PREFIX bundle exec rake db:setup 2>/dev/null; then
+ echo "Warning: unable to set up database." >&2
+ DATABASE_READY=0
+ fi
+ else
+ echo "Warning: Database is not ready to set up. Skipping database setup." >&2
+ DATABASE_READY=0
+ fi
+}
+
+configure_version() {
+ WEB_SERVICE=${WEB_SERVICE:-$(service --status-all 2>/dev/null \
+ | grep -Eo '\bnginx|httpd[^[:space:]]*' || true)}
+ if [ -z "$WEB_SERVICE" ]; then
+ report_web_service_warning "Web service (Nginx or Apache) not found"
+ elif [ "$WEB_SERVICE" != "$(echo "$WEB_SERVICE" | head -n 1)" ]; then
+ WEB_SERVICE=$(echo "$WEB_SERVICE" | head -n 1)
+ report_web_service_warning \
+ "Multiple web services found. Choosing the first one ($WEB_SERVICE)"
+ fi
+
+ if [ -e /etc/redhat-release ]; then
+ # Recognize any service that starts with "nginx"; e.g., nginx16.
+ if [ "$WEB_SERVICE" != "${WEB_SERVICE#nginx}" ]; then
+ WWW_OWNER=nginx
+ else
+ WWW_OWNER=apache
+ fi
+ else
+ # Assume we're on a Debian-based system for now.
+ # Both Apache and Nginx run as www-data by default.
+ WWW_OWNER=www-data
+ fi
+
+ echo
+ echo "Assumption: $WEB_SERVICE is configured to serve Rails from"
+ echo " $RELEASE_PATH"
+ echo "Assumption: $WEB_SERVICE and passenger run as $WWW_OWNER"
+ echo
+
+ echo -n "Creating symlinks to configuration in $CONFIG_PATH ..."
+ setup_confdirs /etc/arvados "$CONFIG_PATH"
+ setup_conffile environments/production.rb environments/production.rb.example \
+ || true
+ setup_conffile application.yml application.yml.example || APPLICATION_READY=0
+ if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
+ setup_conffile database.yml database.yml.example || DATABASE_READY=0
+ fi
+ setup_extra_conffiles
+ echo "... done."
+
+ # Before we do anything else, make sure some directories and files are in place
+ if [ ! -e $SHARED_PATH/log ]; then mkdir -p $SHARED_PATH/log; fi
+ if [ ! -e $RELEASE_PATH/tmp ]; then mkdir -p $RELEASE_PATH/tmp; fi
+ if [ ! -e $RELEASE_PATH/log ]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi
+ if [ ! -e $SHARED_PATH/log/production.log ]; then touch $SHARED_PATH/log/production.log; fi
+
+ cd "$RELEASE_PATH"
+ export RAILS_ENV=production
+
+ if ! $COMMAND_PREFIX bundle --version >/dev/null; then
+ run_and_report "Installing bundle" $COMMAND_PREFIX gem install bundle
+ fi
+
+ run_and_report "Running bundle install" \
+ $COMMAND_PREFIX bundle install --path $SHARED_PATH/vendor_bundle --local --quiet
+
+ echo -n "Ensuring directory and file permissions ..."
+ # Ensure correct ownership of a few files
+ chown "$WWW_OWNER:" $RELEASE_PATH/config/environment.rb
+ chown "$WWW_OWNER:" $RELEASE_PATH/config.ru
+ chown "$WWW_OWNER:" $RELEASE_PATH/Gemfile.lock
+ chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
+ chown -R "$WWW_OWNER:" $SHARED_PATH/log
+ case "$RAILSPKG_DATABASE_LOAD_TASK" in
+ db:schema:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/schema.rb ;;
+ db:structure:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/structure.sql ;;
+ esac
+ chmod 644 $SHARED_PATH/log/*
+ chmod -R 2775 $RELEASE_PATH/tmp
+ echo "... done."
+
+ if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
+ prepare_database
+ fi
+
+ if [ 11 = "$RAILSPKG_SUPPORTS_CONFIG_CHECK$APPLICATION_READY" ]; then
+ run_and_report "Checking application.yml for completeness" \
+ $COMMAND_PREFIX bundle exec rake config:check || APPLICATION_READY=0
+ fi
+
+ # precompile assets; thankfully this does not take long
+ if [ "$APPLICATION_READY" = "1" ]; then
+ run_and_report "Precompiling assets" \
+ $COMMAND_PREFIX bundle exec rake assets:precompile -q -s 2>/dev/null \
+ || APPLICATION_READY=0
+ else
+ echo "Precompiling assets... skipped."
+ fi
+ chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
+
+ setup_before_nginx_restart
+
+ if [ ! -z "$WEB_SERVICE" ]; then
+ service "$WEB_SERVICE" restart
+ fi
+}
+
+if [ "$1" = configure ]; then
+ # This is a debian-based system
+ configure_version
+elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
+ # This is an rpm-based system
+ configure_version
+fi
+
+report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
+report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+set -e
+
+purge () {
+ rm -rf $SHARED_PATH/vendor_bundle
+ rm -rf $SHARED_PATH/log
+ rm -rf $CONFIG_PATH
+ rmdir $SHARED_PATH || true
+ rmdir $INSTALL_PATH || true
+}
+
+if [ "$1" = 'purge' ]; then
+ # This is a debian-based system and purge was requested
+ purge
+elif [ "$1" = "0" ]; then
+ # This is an rpm-based system, no guarantees are made, always purge
+ # Apparently yum doesn't actually remember what it installed.
+ # Clean those files up here, then purge.
+ rm -rf $RELEASE_PATH
+ purge
+fi
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+remove () {
+ rm -f $RELEASE_PATH/config/database.yml
+ rm -f $RELEASE_PATH/config/environments/production.rb
+ rm -f $RELEASE_PATH/config/application.yml
+ # Old API server configuration file.
+ rm -f $RELEASE_PATH/config/initializers/omniauth.rb
+ rm -rf $RELEASE_PATH/public/assets/
+ rm -rf $RELEASE_PATH/tmp
+ rm -rf $RELEASE_PATH/.bundle
+ rm -rf $RELEASE_PATH/log
+}
+
+if [ "$1" = 'remove' ]; then
+ # This is a debian-based system and removal was requested
+ remove
+elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
+ # This is an rpm-based system
+ remove
+fi
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions, before the actual
+# pre/post package work, to set some variable and function defaults.
+
+if [ -z "$INSTALL_PATH" ]; then
+ cat >&2 <<EOF
+
+PACKAGE BUILD ERROR: $0 is missing package metadata.
+
+This package is buggy. Please mail <support@curoverse.com> to let
+us know the name and version number of the package you tried to
+install, and we'll get it fixed.
+
+EOF
+ exit 3
+fi
+
+RELEASE_PATH=$INSTALL_PATH/current
+RELEASE_CONFIG_PATH=$RELEASE_PATH/config
+SHARED_PATH=$INSTALL_PATH/shared
+
+RAILSPKG_SUPPORTS_CONFIG_CHECK=${RAILSPKG_SUPPORTS_CONFIG_CHECK:-1}
+if ! type setup_extra_conffiles >/dev/null 2>&1; then
+ setup_extra_conffiles() { return; }
+fi
+if ! type setup_before_nginx_restart >/dev/null 2>&1; then
+ setup_before_nginx_restart() { return; }
+fi
--- /dev/null
+#!/bin/bash
+
+function usage {
+ echo >&2
+ echo >&2 "usage: $0 [options]"
+ echo >&2
+ echo >&2 "$0 options:"
+ echo >&2 " -t, --tags [csv_tags] comma separated tags"
+ echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2
+ echo >&2 " If no options are given, just builds the images."
+}
+
+upload=false
+
+# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+TEMP=`getopt -o hut: \
+ --long help,upload,tags: \
+ -n "$0" -- "$@"`
+
+if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while [ $# -ge 1 ]
+do
+ case $1 in
+ -u | --upload)
+ upload=true
+ shift
+ ;;
+ -t | --tags)
+ case "$2" in
+ "")
+ echo "ERROR: --tags needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ tags=$2;
+ shift 2
+ ;;
+ esac
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+
+EXITCODE=0
+
+COLUMNS=80
+
+title () {
+ printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
+}
+
+docker_push () {
+ if [[ ! -z "$tags" ]]
+ then
+ for tag in $( echo $tags|tr "," " " )
+ do
+ $DOCKER tag $1 $1:$tag
+ done
+ fi
+
+ # Sometimes docker push fails; retry it a few times if necessary.
+ for i in `seq 1 5`; do
+ $DOCKER push $*
+ ECODE=$?
+ if [[ "$ECODE" == "0" ]]; then
+ break
+ fi
+ done
+
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker push $* failed !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+echo $WORKSPACE
+
+# find the docker binary
+DOCKER=`which docker.io`
+
+if [[ "$DOCKER" == "" ]]; then
+ DOCKER=`which docker`
+fi
+
+if [[ "$DOCKER" == "" ]]; then
+ title "Error: you need to have docker installed. Could not find the docker executable."
+ exit 1
+fi
+
+# DOCKER
+title "Starting docker build"
+
+timer_reset
+
+# clean up the docker build environment
+cd "$WORKSPACE"
+
+tools/arvbox/bin/arvbox build dev
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+tools/arvbox/bin/arvbox build localdemo
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+title "docker build complete (`timer`)"
+
+title "uploading images"
+
+timer_reset
+
+if [[ "$ECODE" != "0" ]]; then
+ title "upload arvados images SKIPPED because build failed"
+else
+ if [[ $upload == true ]]; then
+ ## 20150526 nico -- *sometimes* dockerhub needs re-login
+ ## even though credentials are already in .dockercfg
+ docker login -u arvados
+
+ docker_push arvados/arvbox-dev
+ docker_push arvados/arvbox-demo
+ title "upload arvados images complete (`timer`)"
+ else
+ title "upload arvados images SKIPPED because no --upload option set"
+ fi
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+function usage {
+ echo >&2
+ echo >&2 "usage: $0 [options]"
+ echo >&2
+ echo >&2 "$0 options:"
+ echo >&2 " -t, --tags [csv_tags] comma separated tags"
+ echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2
+ echo >&2 " If no options are given, just builds the images."
+}
+
+upload=false
+
+# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+TEMP=`getopt -o hut: \
+ --long help,upload,tags: \
+ -n "$0" -- "$@"`
+
+if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while [ $# -ge 1 ]
+do
+ case $1 in
+ -u | --upload)
+ upload=true
+ shift
+ ;;
+ -t | --tags)
+ case "$2" in
+ "")
+ echo "ERROR: --tags needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ tags=$2;
+ shift 2
+ ;;
+ esac
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+
+EXITCODE=0
+
+COLUMNS=80
+
+title () {
+ printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
+}
+
+docker_push () {
+ if [[ ! -z "$tags" ]]
+ then
+ for tag in $( echo $tags|tr "," " " )
+ do
+ $DOCKER tag -f $1 $1:$tag
+ done
+ fi
+
+ # Sometimes docker push fails; retry it a few times if necessary.
+ for i in `seq 1 5`; do
+ $DOCKER push $*
+ ECODE=$?
+ if [[ "$ECODE" == "0" ]]; then
+ break
+ fi
+ done
+
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker push $* failed !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+echo $WORKSPACE
+
+# find the docker binary
+DOCKER=`which docker.io`
+
+if [[ "$DOCKER" == "" ]]; then
+ DOCKER=`which docker`
+fi
+
+if [[ "$DOCKER" == "" ]]; then
+ title "Error: you need to have docker installed. Could not find the docker executable."
+ exit 1
+fi
+
+# DOCKER
+title "Starting docker build"
+
+timer_reset
+
+# clean up the docker build environment
+cd "$WORKSPACE"
+cd docker
+rm -f jobs-image
+rm -f config.yml
+
+# Get test config.yml file
+cp $HOME/docker/config.yml .
+
+./build.sh jobs-image
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+title "docker build complete (`timer`)"
+
+title "uploading images"
+
+timer_reset
+
+if [[ "$ECODE" != "0" ]]; then
+ title "upload arvados images SKIPPED because build failed"
+else
+ if [[ $upload == true ]]; then
+ ## 20150526 nico -- *sometimes* dockerhub needs re-login
+ ## even though credentials are already in .dockercfg
+ docker login -u arvados
+
+ docker_push arvados/jobs
+ title "upload arvados images complete (`timer`)"
+ else
+ title "upload arvados images SKIPPED because no --upload option set"
+ fi
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Orchestrate run-build-packages.sh for every target
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+Options:
+
+--command
+ Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install tests
+--debug
+ Output debug information (default: false)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+set -e
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,test-packages,debug,command:,only-test: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+COMMAND=
+DEBUG=
+TEST_PACKAGES=
+ONLY_TEST=
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --debug)
+ DEBUG="--debug"
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --test-packages)
+ TEST_PACKAGES="--test-packages"
+ ;;
+ --only-test)
+ ONLY_TEST="$1 $2"; shift
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+cd $(dirname $0)
+
+FINAL_EXITCODE=0
+
+for dockerfile_path in $(find -name Dockerfile); do
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ true
+ else
+ FINAL_EXITCODE=$?
+ fi
+done
+
+if test $FINAL_EXITCODE != 0 ; then
+ echo "Build packages failed with code $FINAL_EXITCODE" >&2
+fi
+
+exit $FINAL_EXITCODE
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Orchestrate run-build-packages.sh for one target
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+--target <target>
+ Distribution to build packages for (default: debian7)
+--command
+ Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install test script "test-packages-$target.sh"
+--debug
+ Output debug information (default: false)
+--only-test
+ Test only a specific package
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+set -e
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,debug,test-packages,target:,command:,only-test: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+TARGET=debian7
+COMMAND=
+DEBUG=
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --only-test)
+ packages="$2"; shift
+ ;;
+ --debug)
+ DEBUG=" --debug"
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --test-packages)
+ test_packages=1
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+set -e
+
+if [[ -n "$test_packages" ]]; then
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+ createrepo $WORKSPACE/packages/$TARGET
+ fi
+
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+ (cd $WORKSPACE/packages/$TARGET
+ dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz
+ )
+ fi
+
+ COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
+ IMAGE="arvados/package-test:$TARGET"
+else
+ IMAGE="arvados/build:$TARGET"
+ if [[ "$COMMAND" != "" ]]; then
+ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
+ fi
+fi
+
+JENKINS_DIR=$(dirname "$(readlink -e "$0")")
+
+if [[ -n "$test_packages" ]]; then
+ pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+ pushd "$JENKINS_DIR/package-build-dockerfiles"
+ make "$TARGET/generated"
+fi
+
+echo $TARGET
+cd $TARGET
+time docker build --tag=$IMAGE .
+popd
+
+if test -z "$packages" ; then
+ packages="arvados-api-server
+ arvados-data-manager
+ arvados-docker-cleaner
+ arvados-git-httpd
+ arvados-node-manager
+ arvados-src
+ arvados-workbench
+ crunchstat
+ keepproxy
+ keep-rsync
+ keepstore
+ keep-web
+ libarvados-perl"
+
+ case "$TARGET" in
+ centos6)
+ packages="$packages python27-python-arvados-fuse
+ python27-python-arvados-python-client"
+ ;;
+ *)
+ packages="$packages python-arvados-fuse
+ python-arvados-python-client"
+ ;;
+ esac
+fi
+
+FINAL_EXITCODE=0
+
+package_fails=""
+
+mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
+mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
+
+docker_volume_args=(
+ -v "$JENKINS_DIR:/jenkins"
+ -v "$WORKSPACE:/arvados"
+ -v /arvados/services/api/vendor/bundle
+ -v /arvados/apps/workbench/vendor/bundle
+ -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
+ -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
+)
+
+if [[ -n "$test_packages" ]]; then
+ for p in $packages ; do
+ echo
+ echo "START: $p test on $IMAGE" >&2
+ if docker run --rm \
+ "${docker_volume_args[@]}" \
+ --env ARVADOS_DEBUG=1 \
+ --env "TARGET=$TARGET" \
+ --env "WORKSPACE=/arvados" \
+ "$IMAGE" $COMMAND $p
+ then
+ echo "OK: $p test on $IMAGE succeeded" >&2
+ else
+ FINAL_EXITCODE=$?
+ package_fails="$package_fails $p"
+ echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
+ fi
+ done
+else
+ echo
+ echo "START: build packages on $IMAGE" >&2
+ if docker run --rm \
+ "${docker_volume_args[@]}" \
+ --env ARVADOS_DEBUG=1 \
+ "$IMAGE" $COMMAND
+ then
+ echo
+ echo "OK: build packages on $IMAGE succeeded" >&2
+ else
+ FINAL_EXITCODE=$?
+ echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
+ fi
+fi
+
+if test -n "$package_fails" ; then
+ echo "Failed package tests:$package_fails" >&2
+fi
+
+exit $FINAL_EXITCODE
--- /dev/null
+#!/bin/bash
+
+JENKINS_DIR=$(dirname $(readlink -e "$0"))
+. "$JENKINS_DIR/run-library.sh"
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Build Arvados SSO server package
+
+Syntax:
+ WORKSPACE=/path/to/arvados-sso $(basename $0) [options]
+
+Options:
+
+--debug
+ Output debug information (default: false)
+--target
+ Distribution to build packages for (default: debian7)
+
+WORKSPACE=path Path to the Arvados SSO source tree to build packages from
+
+EOF
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+TARGET=debian7
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,build-bundle-packages,debug,target: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --debug)
+ DEBUG=1
+ ;;
+ --test-packages)
+ test_packages=1
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+case "$TARGET" in
+ debian7)
+ FORMAT=deb
+ ;;
+ debian8)
+ FORMAT=deb
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ ;;
+ ubuntu1404)
+ FORMAT=deb
+ ;;
+ centos6)
+ FORMAT=rpm
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+# Test for fpm
+fpm --version >/dev/null 2>&1
+
+if [[ "$?" != 0 ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: fpm not found"
+ echo >&2
+ exit 1
+fi
+
+RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
+RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
+if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
+debug_echo "Workspace is $WORKSPACE"
+
+if [[ -f /etc/profile.d/rvm.sh ]]; then
+ source /etc/profile.d/rvm.sh
+ GEM="rvm-exec default gem"
+else
+ GEM=gem
+fi
+
+# Make all files world-readable -- jenkins runs with umask 027, and has checked
+# out our git tree here
+chmod o+r "$WORKSPACE" -R
+
+# More cleanup - make sure all executables that we'll package are 755
+# No executables in the sso server package
+#find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
+
+# Now fix our umask to something better suited to building and publishing
+# gems and packages
+umask 0022
+
+debug_echo "umask is" `umask`
+
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p "$WORKSPACE/packages/$TARGET"
+fi
+
+# Build the SSO server package
+handle_rails_package arvados-sso-server "$WORKSPACE" \
+ "$WORKSPACE/LICENCE" --url="https://arvados.org" \
+ --description="Arvados SSO server - Arvados is a free and open source platform for big data science." \
+ --license="Expat license"
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+. `dirname "$(readlink -f "$0")"`/run-library.sh
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Build Arvados packages
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+Options:
+
+--build-bundle-packages (default: false)
+ Build api server and workbench packages with vendor/bundle included
+--debug
+ Output debug information (default: false)
+--target
+ Distribution to build packages for (default: debian7)
+--command
+ Build command to execute (defaults to the run command defined in the
+ Docker image)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+TARGET=debian7
+COMMAND=
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,build-bundle-packages,debug,target: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --debug)
+ DEBUG=1
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+if [[ "$COMMAND" != "" ]]; then
+ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+fi
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
+
+PYTHON2_VERSION=2.7
+PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+
+case "$TARGET" in
+ debian7)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ 'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' 'rdflib-jsonld>=0.3.0' \
+ shellescape mistune)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ debian8)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' 'rdflib-jsonld>=0.3.0' \
+ shellescape mistune)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
+ 'rdflib-jsonld>=0.3.0' shellescape mistune)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ ubuntu1404)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
+ google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
+ rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
+ 'rdflib-jsonld>=0.3.0' shellescape mistune)
+ PYTHON3_BACKPORTS=(docker-py requests websocket-client)
+ ;;
+ centos6)
+ FORMAT=rpm
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON_BACKPORTS=(python-gflags google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
+ python-daemon lockfile llfuse==0.41.1 'pbr<1.0' pyyaml \
+ 'rdflib>=4.2.0' 'rdflib-jsonld>=0.3.0' shellescape mistune)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ export PYCURL_SSL_LIBRARY=nss
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+# Test for fpm
+fpm --version >/dev/null 2>&1
+
+if [[ "$?" != 0 ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: fpm not found"
+ echo >&2
+ exit 1
+fi
+
+EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
+EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
+
+RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
+RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
+if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
+debug_echo "Workspace is $WORKSPACE"
+
+if [[ -f /etc/profile.d/rvm.sh ]]; then
+ source /etc/profile.d/rvm.sh
+ GEM="rvm-exec default gem"
+else
+ GEM=gem
+fi
+
+# Make all files world-readable -- jenkins runs with umask 027, and has checked
+# out our git tree here
+chmod o+r "$WORKSPACE" -R
+
+# More cleanup - make sure all executables that we'll package are 755
+find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
+
+# Now fix our umask to something better suited to building and publishing
+# gems and packages
+umask 0022
+
+debug_echo "umask is" `umask`
+
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p $WORKSPACE/packages/$TARGET
+fi
+
+# Perl packages
+debug_echo -e "\nPerl packages\n"
+
+cd "$WORKSPACE/sdk/perl"
+
+if [[ -e Makefile ]]; then
+ make realclean >"$STDOUT_IF_DEBUG"
+fi
+find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+ -delete
+rm -rf install
+
+perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+ fpm_build install/lib/=/usr/share libarvados-perl \
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+ "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+
+# Ruby gems
+debug_echo -e "\nRuby gems\n"
+
+FPM_GEM_PREFIX=$($GEM environment gemdir)
+
+cd "$WORKSPACE/sdk/ruby"
+handle_ruby_gem arvados
+
+cd "$WORKSPACE/sdk/cli"
+handle_ruby_gem arvados-cli
+
+cd "$WORKSPACE/services/login-sync"
+handle_ruby_gem arvados-login-sync
+
+# Python packages
+debug_echo -e "\nPython packages\n"
+
+cd "$WORKSPACE/sdk/pam"
+handle_python_package
+
+cd "$WORKSPACE/sdk/python"
+handle_python_package
+
+cd "$WORKSPACE/sdk/cwl"
+handle_python_package
+
+cd "$WORKSPACE/services/fuse"
+handle_python_package
+
+cd "$WORKSPACE/services/nodemanager"
+handle_python_package
+
+# arvados-src
+(
+ set -e
+
+ cd "$WORKSPACE"
+ COMMIT_HASH=$(format_last_commit_here "%H")
+
+ SRC_BUILD_DIR=$(mktemp -d)
+ # mktemp creates the directory with 0700 permissions by default
+ chmod 755 $SRC_BUILD_DIR
+ git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+ cd "$SRC_BUILD_DIR"
+
+ # go into detached-head state
+ git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+ echo "$COMMIT_HASH" >git-commit.version
+
+ cd "$SRC_BUILD_DIR"
+ PKG_VERSION=$(version_from_git)
+ cd $WORKSPACE/packages/$TARGET
+ fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+ rm -rf "$SRC_BUILD_DIR"
+)
+
+# On older platforms we need to publish a backport of libfuse >=2.9.2,
+# and we need to build and install it here in order to even build an
+# llfuse package.
+cd $WORKSPACE/packages/$TARGET
+if [[ $TARGET =~ ubuntu1204 ]]; then
+ # port libfuse 2.9.2 to Ubuntu 12.04
+ LIBFUSE_DIR=$(mktemp -d)
+ (
+ cd $LIBFUSE_DIR
+ # download fuse 2.9.2 ubuntu 14.04 source package
+ file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+ file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+ file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+
+ # install dpkg-source and dpkg-buildpackage commands
+ apt-get install -y --no-install-recommends dpkg-dev
+
+ # extract source and apply patches
+ dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
+ rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
+
+ # add new version to changelog
+ cd fuse-2.9.2
+ (
+ echo "fuse (2.9.2-5) precise; urgency=low"
+ echo
+ echo " * Backported from trusty-security to precise"
+ echo
+ echo " -- Joshua Randall <jcrandall@alum.mit.edu> Thu, 4 Feb 2016 11:31:00 -0000"
+ echo
+ cat debian/changelog
+ ) > debian/changelog.new
+ mv debian/changelog.new debian/changelog
+
+ # install build-deps and build
+ apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
+ dpkg-buildpackage -rfakeroot -b
+ )
+ fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
+ dpkg -i \
+ "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
+ "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
+ "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
+ apt-get -y --no-install-recommends -f install
+ rm -rf $LIBFUSE_DIR
+elif [[ $TARGET =~ centos6 ]]; then
+ # port fuse 2.9.2 to centos 6
+ # install tools to build rpm from source
+ yum install -y rpm-build redhat-rpm-config
+ LIBFUSE_DIR=$(mktemp -d)
+ (
+ cd "$LIBFUSE_DIR"
+ # download fuse 2.9.2 centos 7 source rpm
+ file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
+ (
+ # modify source rpm spec to remove conflict on filesystem version
+ mkdir -p /root/rpmbuild/SOURCES
+ cd /root/rpmbuild/SOURCES
+ rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
+ perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
+ )
+ # build rpms from source
+ rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
+ rm -f fuse-2.9.2-6.el7.src.rpm
+ # move built RPMs to LIBFUSE_DIR
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ rm -rf /root/rpmbuild
+ )
+ fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+ fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+ yum install -y \
+ "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
+ "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
+ "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
+fi
+
+# Go binaries
+cd $WORKSPACE/packages/$TARGET
+export GOPATH=$(mktemp -d)
+package_go_binary services/keepstore keepstore \
+ "Keep storage daemon, accessible to clients on the LAN"
+package_go_binary services/keepproxy keepproxy \
+ "Make a Keep cluster accessible to clients that are not on the LAN"
+package_go_binary services/keep-web keep-web \
+ "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary services/datamanager arvados-data-manager \
+ "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary services/arv-git-httpd arvados-git-httpd \
+ "Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/crunchstat crunchstat \
+ "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary tools/keep-rsync keep-rsync \
+ "Copy all data from one set of Keep servers to another"
+
+# The Python SDK
+# Please resist the temptation to add --no-python-fix-name to the fpm call here
+# (which would remove the python- prefix from the package name), because this
+# package is a dependency of arvados-fuse, and fpm can not omit the python-
+# prefix from only one of the dependencies of a package... Maybe I could
+# whip up a patch and send it upstream, but that will be for another day. Ward,
+# 2014-05-15
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/sdk/python/build"
+fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
+
+# cwl-runner
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/sdk/cwl/build"
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
+
+# schema_salad. This is a python dependency of arvados-cwl-runner,
+# but we can't use the usual PYTHONPACKAGES way to build this package due to the
+# intricacies of how version numbers get generated in setup.py: we need version
+# 1.7.20160316203940. If we don't explicitly list that version with the -v
+# argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get
+# a package with version 1.7. That's because our gittagger hack is not being
+# picked up by self.distribution.get_version(), which is called from
+# https://github.com/jordansissel/fpm/blob/master/lib/fpm/package/pyfpm/get_metadata.py
+# by means of this command:
+#
+# python2.7 setup.py --command-packages=pyfpm get_metadata --output=metadata.json
+#
+# So we build this thing separately.
+#
+# Ward, 2016-03-17
+fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t deb --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n python-schema-salad --iteration 1 --python-bin python2.7 --python-easyinstall easy_install-2.7 --python-package-name-prefix python --depends python2.7 -v 1.7.20160316203940 schema_salad
+
+# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
+fpm --maintainer='Ward Vandewege <ward@curoverse.com>' -s python -t deb --exclude=*/dist-packages/tests/* --exclude=*/site-packages/tests/* --deb-ignore-iteration-in-dependencies -n python-cwltool --iteration 1 --python-bin python2.7 --python-easyinstall easy_install-2.7 --python-package-name-prefix python --depends python2.7 -v 1.0.20160316204054 cwltool
+
+# The PAM module
+if [[ $TARGET =~ debian|ubuntu ]]; then
+ cd $WORKSPACE/packages/$TARGET
+ rm -rf "$WORKSPACE/sdk/pam/build"
+ fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+fi
+
+# The FUSE driver
+# Please see comment about --no-python-fix-name above; we stay consistent and do
+# not omit the python- prefix first.
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/fuse/build"
+fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+
+# The node manager
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/nodemanager/build"
+fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+
+# The Docker image cleaner
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/dockercleaner/build"
+fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+
+# Forked libcloud
+LIBCLOUD_DIR=$(mktemp -d)
+(
+ cd $LIBCLOUD_DIR
+ git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+ git checkout apache-libcloud-$LIBCLOUD_PIN
+ # libcloud is absurdly noisy without -q, so force -q here
+ OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+ DASHQ_UNLESS_DEBUG=-q
+ handle_python_package
+ DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+)
+fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+rm -rf $LIBCLOUD_DIR
+
+# Python 2 dependencies
+declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
+# Add --no-use-wheel if this pip knows it.
+pip wheel --help >/dev/null 2>&1
+case "$?" in
+ 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
+ 2) ;;
+ *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
+esac
+
+for deppkg in "${PYTHON_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ case "$deppkg" in
+ httplib2|google-api-python-client)
+ # Work around 0640 permissions on some package files.
+ # See #7591 and #7991.
+ pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
+ set -e
+ cd "$pyfpm_workdir"
+ pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+ tar -xf "$deppkg"-*.tar*
+ cd "$deppkg"-*/
+ "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
+ chmod -R go+rX .
+ set +e
+ # --iteration 2 provides an upgrade for previously built
+ # buggy packages.
+ fpm_build . "$outname" "" python "" --iteration 2
+ # The upload step uses the package timestamp to determine
+ # whether it's new. --no-clobber plays nice with that.
+ mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
+ )
+ if [ 0 != "$?" ]; then
+ echo "ERROR: $deppkg build process failed"
+ EXITCODE=1
+ fi
+ if [ -n "$pyfpm_workdir" ]; then
+ rm -rf "$pyfpm_workdir"
+ fi
+ ;;
+ *)
+ fpm_build "$deppkg" "$outname"
+ ;;
+ esac
+done
+
+# Python 3 dependencies
+for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+ # The empty string is the vendor argument: these aren't Curoverse software.
+ fpm_build "$deppkg" "$outname" "" python3
+done
+
+# Build the API server package
+handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados API server - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0"
+
+# Build the workbench server package
+(
+ set -e
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up application.yml and production.rb so that asset precompilation works
+ \cp config/application.yml.example config/application.yml -f
+ \cp config/environments/production.rb.example config/environments/production.rb -f
+ sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+ RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+ # Remove generated configuration files so they don't go in the package.
+ rm config/application.yml config/environments/production.rb
+)
+
+if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0"
+fi
+
+# clean up temporary GOPATH
+rm -rf "$GOPATH"
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Build, test and (optionally) upload packages for one target
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+--target <target>
+ Distribution to build packages for (default: debian7)
+--upload
+ If the build and test steps are successful, upload the packages
+ to a remote apt repository (default: false)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,upload,target: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+TARGET=debian7
+UPLOAD=0
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --upload)
+ UPLOAD=1
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+exit_cleanly() {
+ trap - INT
+ report_outcomes
+ exit ${#failures}
+}
+
+COLUMNS=80
+. $WORKSPACE/build/run-library.sh
+
+title "Start build packages"
+timer_reset
+
+$WORKSPACE/build/run-build-packages-one-target.sh --target $TARGET
+
+checkexit $? "build packages"
+title "End of build packages (`timer`)"
+
+title "Start test packages"
+timer_reset
+
+if [ ${#failures[@]} -eq 0 ]; then
+ $WORKSPACE/build/run-build-packages-one-target.sh --target $TARGET --test-packages
+else
+ echo "Skipping package upload, there were errors building the packages"
+fi
+
+checkexit $? "test packages"
+title "End of test packages (`timer`)"
+
+if [[ "$UPLOAD" != 0 ]]; then
+ title "Start upload packages"
+ timer_reset
+
+ if [ ${#failures[@]} -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ else
+ echo "Skipping package upload, there were errors building and/or testing the packages"
+ fi
+ checkexit $? "upload packages"
+ title "End of upload packages (`timer`)"
+fi
+
+exit_cleanly
--- /dev/null
+#!/bin/bash
+
+# A library of functions shared by the various scripts in this directory.
+
+# This is the timestamp about when we merged changed to include licenses
+# with Arvados packages. We use it as a heuristic to add revisions for
+# older packages.
+LICENSE_PACKAGE_TS=20151208015500
+
+debug_echo () {
+ echo "$@" >"$STDOUT_IF_DEBUG"
+}
+
+find_easy_install() {
+ for version_suffix in "$@"; do
+ if "easy_install$version_suffix" --version >/dev/null 2>&1; then
+ echo "easy_install$version_suffix"
+ return 0
+ fi
+ done
+ cat >&2 <<EOF
+$helpmessage
+
+Error: easy_install$1 (from Python setuptools module) not found
+
+EOF
+ exit 1
+}
+
+format_last_commit_here() {
+ local format="$1"; shift
+ TZ=UTC git log -n1 --first-parent "--format=format:$format" .
+}
+
+version_from_git() {
+ # Generates a version number from the git log for the current working
+ # directory, and writes it to stdout.
+ local git_ts git_hash
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+}
+
+nohash_version_from_git() {
+ version_from_git | cut -d. -f1-3
+}
+
+timestamp_from_git() {
+ format_last_commit_here "%ct"
+}
+
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ # This package doesn't need rebuilding.
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
+handle_ruby_gem() {
+ local gem_name="$1"; shift
+ local gem_version="$(nohash_version_from_git)"
+ local gem_src_dir="$(pwd)"
+
+ if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
+ find -maxdepth 1 -name "${gem_name}-*.gem" -delete
+
+ # -q appears to be broken in gem version 2.2.2
+ $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
+ fi
+}
+
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
+package_go_binary() {
+ local src_path="$1"; shift
+ local prog="$1"; shift
+ local description="$1"; shift
+ local license_file="${1:-agpl-3.0.txt}"; shift
+
+ debug_echo "package_go_binary $src_path as $prog"
+
+ local basename="${src_path##*/}"
+
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
+ local version="$(version_from_git)"
+ local timestamp="$(timestamp_from_git)"
+
+ # If the command imports anything from the Arvados SDK, bump the
+ # version number and build a new package whenever the SDK changes.
+ if grep -qr git.curoverse.com/arvados .; then
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
+ if [[ $(timestamp_from_git) -gt "$timestamp" ]]; then
+ version=$(version_from_git)
+ fi
+ fi
+
+ cd $WORKSPACE/packages/$TARGET
+ go get "git.curoverse.com/arvados.git/$src_path"
+ fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file"
+}
+
+default_iteration() {
+ local package_name="$1"; shift
+ local package_version="$1"; shift
+ local iteration=1
+ if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
+ [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
+ iteration=2
+ fi
+ echo $iteration
+}
+
+_build_rails_package_scripts() {
+ local pkgname="$1"; shift
+ local destdir="$1"; shift
+ local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
+ for scriptname in postinst prerm postrm; do
+ cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
+ >"$destdir/$scriptname" || return $?
+ done
+}
+
+handle_rails_package() {
+ local pkgname="$1"; shift
+ local srcdir="$1"; shift
+ local license_path="$1"; shift
+ local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
+ local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ set -e
+ _build_rails_package_scripts "$pkgname" "$scripts_dir"
+ cd "$srcdir"
+ mkdir -p tmp
+ version_from_git >"$version_file"
+ git rev-parse HEAD >git-commit.version
+ bundle package --all
+ )
+ if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
+ echo "ERROR: $pkgname package prep failed" >&2
+ rm -rf "$scripts_dir" "$version_file"
+ EXITCODE=1
+ return 1
+ fi
+ local railsdir="/var/www/${pkgname%-server}/current"
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
+ "$(cat "$version_file")")
+ local license_arg="$license_path=$railsdir/$(basename "$license_path")"
+ # --iteration=5 accommodates the package script bugfixes #8371 and #8413.
+ local -a switches=(--iteration=5
+ --after-install "$scripts_dir/postinst"
+ --before-remove "$scripts_dir/prerm"
+ --after-remove "$scripts_dir/postrm")
+ # For some reason fpm excludes need to not start with /.
+ local exclude_root="${railsdir#/}"
+ # .git and packages are for the SSO server, which is built from its
+ # repository root.
+ local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+ config/deploy\* config/application.yml)
+ # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
+ if [[ "$pkgname" != "arvados-workbench" ]]; then
+ exclude_list+=('config/database.yml')
+ fi
+ for exclude in ${exclude_list[@]}; do
+ switches+=(-x "$exclude_root/$exclude")
+ done
+ fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
+ rm -rf "$scripts_dir" "$version_file"
+}
+
+# Build packages for everything
+fpm_build () {
+ # The package source. Depending on the source type, this can be a
+ # path, or the name of the package in an upstream repository (e.g.,
+ # pip).
+ PACKAGE=$1
+ shift
+ # The name of the package to build. Defaults to $PACKAGE.
+ PACKAGE_NAME=${1:-$PACKAGE}
+ shift
+ # Optional: the vendor of the package. Should be "Curoverse, Inc." for
+ # packages of our own software. Passed to fpm --vendor.
+ VENDOR=$1
+ shift
+ # The type of source package. Passed to fpm -s. Default "python".
+ PACKAGE_TYPE=${1:-python}
+ shift
+ # Optional: the package version number. Passed to fpm -v.
+ VERSION=$1
+ shift
+
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ set -- "$@" --python-bin python2.7 \
+ --python-easyinstall "$EASY_INSTALL2" \
+ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --depends "$PYTHON2_PACKAGE"
+ ;;
+ python3)
+ # fpm does not actually support a python3 package type. Instead
+ # we recognize it as a convenience shortcut to add several
+ # necessary arguments to fpm's command line later, after we're
+ # done handling positional arguments.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 \
+ --python-easyinstall "$EASY_INSTALL3" \
+ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --depends "$PYTHON3_PACKAGE"
+ ;;
+ esac
+
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ if [ python = "$PACKAGE_TYPE" ]; then
+ COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
+ if [ deb = "$FORMAT" ]; then
+ # Dependencies are built from setup.py. Since setup.py will never
+ # refer to Debian package iterations, it doesn't make sense to
+ # enforce those in the .deb dependencies.
+ COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
+ fi
+ fi
+
+ if [[ "${DEBUG:-0}" != "0" ]]; then
+ COMMAND_ARR+=('--verbose' '--log' 'info')
+ fi
+
+ if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
+ COMMAND_ARR+=('-n' "$PACKAGE_NAME")
+ fi
+
+ if [[ "$VENDOR" != "" ]]; then
+ COMMAND_ARR+=('--vendor' "$VENDOR")
+ fi
+
+ if [[ "$VERSION" != "" ]]; then
+ COMMAND_ARR+=('-v' "$VERSION")
+ fi
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")")
+
+ # Append --depends X and other arguments specified by fpm-info.sh in
+ # the package source dir. These are added last so they can override
+ # the arguments added by this script.
+ declare -a fpm_args=()
+ declare -a build_depends=()
+ declare -a fpm_depends=()
+ declare -a fpm_exclude=()
+ declare -a fpm_dirs=(
+ # source dir part of 'dir' package ("/source=/dest" => "/source"):
+ "${PACKAGE%%=/*}"
+ # backports ("llfuse==0.41.1" => "backports/python-llfuse")
+ "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
+ for pkgdir in "${fpm_dirs[@]}"; do
+ fpminfo="$pkgdir/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ debug_echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ break
+ fi
+ done
+ for pkg in "${build_depends[@]}"; do
+ if [[ $TARGET =~ debian|ubuntu ]]; then
+ pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
+ if [[ -e $pkg_deb ]]; then
+ echo "Installing build_dep $pkg from $pkg_deb"
+ dpkg -i "$pkg_deb"
+ else
+ echo "Attemping to install build_dep $pkg using apt-get"
+ apt-get install -y "$pkg"
+ fi
+ apt-get -y -f install
+ else
+ pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
+ if [[ -e $pkg_rpm ]]; then
+ echo "Installing build_dep $pkg from $pkg_rpm"
+ rpm -i "$pkg_rpm"
+ else
+ echo "Attemping to install build_dep $pkg"
+ rpm -i "$pkg"
+ fi
+ fi
+ done
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--depends' "$i")
+ done
+ for i in "${fpm_exclude[@]}"; do
+ COMMAND_ARR+=('--exclude' "$i")
+ done
+
+ # Append remaining function arguments directly to fpm's command line.
+ for i; do
+ COMMAND_ARR+=("$i")
+ done
+
+ COMMAND_ARR+=("${fpm_args[@]}")
+
+ COMMAND_ARR+=("$PACKAGE")
+
+ debug_echo -e "\n${COMMAND_ARR[@]}\n"
+
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+
+ fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+}
+
+# verify build results
+fpm_verify () {
+ FPM_EXIT_CODE=$1
+ shift
+ FPM_RESULTS=$@
+
+ FPM_PACKAGE_NAME=''
+ if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
+ FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
+ fi
+
+ if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
+ EXITCODE=1
+ echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
+ echo
+ echo $FPM_RESULTS
+ echo
+ elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
+ echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
+ elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
+ echo "Error building package for $1:\n $FPM_RESULTS"
+ fi
+}
+
+install_package() {
+ PACKAGES=$@
+ if [[ "$FORMAT" == "deb" ]]; then
+ $SUDO apt-get install $PACKAGES --yes
+ elif [[ "$FORMAT" == "rpm" ]]; then
+ $SUDO yum -q -y install $PACKAGES
+ fi
+}
+
+title () {
+ txt="********** $1 **********"
+ printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
+}
+
+checkexit() {
+ if [[ "$1" != "0" ]]; then
+ title "!!!!!! $2 FAILED !!!!!!"
+ failures+=("$2 (`timer`)")
+ else
+ successes+=("$2 (`timer`)")
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+report_outcomes() {
+ for x in "${successes[@]}"
+ do
+ echo "Pass: $x"
+ done
+
+ if [[ ${#failures[@]} == 0 ]]
+ then
+ echo "All test suites passed."
+ else
+ echo "Failures (${#failures[@]}):"
+ for x in "${failures[@]}"
+ do
+ echo "Fail: $x"
+ done
+ fi
+}
+
--- /dev/null
+#!/bin/bash
+
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
+
+COLUMNS=80
+. `dirname "$(readlink -f "$0")"`/run-library.sh
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Install and test Arvados components.
+
+Exit non-zero if any tests fail.
+
+Syntax:
+ $(basename $0) WORKSPACE=/path/to/arvados [options]
+
+Options:
+
+--skip FOO Do not test the FOO component.
+--only FOO Do not test anything except the FOO component.
+--temp DIR Install components and dependencies under DIR instead of
+ making a new temporary directory. Implies --leave-temp.
+--leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
+ Instead, show the path to give as --temp to reuse them in
+ subsequent invocations.
+--skip-install Do not run any install steps. Just run tests.
+ You should provide GOPATH, GEMHOME, and VENVDIR options
+ from a previous invocation if you use this option.
+--only-install Run specific install step
+WORKSPACE=path Arvados source tree to test.
+CONFIGSRC=path Dir with api server config files to copy into source tree.
+ (If none given, leave config files alone in source tree.)
+services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
+ Restrict apiserver tests to the given file
+sdk/python_test="--test-suite test.test_keep_locator"
+ Restrict Python SDK tests to the given class
+apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
+ Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+ Show all log messages, even when tests pass (also works
+ with services/keepstore_test etc.)
+ARVADOS_DEBUG=1
+ Print more debug messages
+envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
+ *_test, and other examples shown above.
+
+Assuming --skip-install is not given, all components are installed
+into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
+test suites depend on other components being installed, and installing
+everything tends to be quicker than debugging dependencies.
+
+As a special concession to the current CI server config, CONFIGSRC
+defaults to $HOME/arvados-api-server if that directory exists.
+
+More information and background:
+
+https://arvados.org/projects/arvados/wiki/Running_tests
+
+Available tests:
+
+apps/workbench
+apps/workbench_benchmark
+apps/workbench_profile
+doc
+services/api
+services/arv-git-httpd
+services/crunchstat
+services/dockercleaner
+services/fuse
+services/keep-web
+services/keepproxy
+services/keepstore
+services/login-sync
+services/nodemanager
+services/crunch-run
+services/crunch-dispatch-local
+services/crunch-dispatch-slurm
+sdk/cli
+sdk/pam
+sdk/python
+sdk/ruby
+sdk/go/arvadosclient
+sdk/go/keepclient
+sdk/go/manifest
+sdk/go/blockdigest
+sdk/go/streamer
+sdk/go/crunchrunner
+sdk/cwl
+tools/crunchstat-summary
+tools/keep-rsync
+
+EOF
+
+# First make sure to remove any ARVADOS_ variables from the calling
+# environment that could interfere with the tests.
+unset $(env | cut -d= -f1 | grep \^ARVADOS_)
+
+# Reset other variables that could affect our [tests'] behavior by
+# accident.
+GITDIR=
+GOPATH=
+VENVDIR=
+VENV3DIR=
+PYTHONPATH=
+GEMHOME=
+PERLINSTALLBASE=
+
+skip_install=
+temp=
+temp_preserve=
+
+clear_temp() {
+ if [[ -z "$temp" ]]; then
+ # we didn't even get as far as making a temp dir
+ :
+ elif [[ -z "$temp_preserve" ]]; then
+ rm -rf "$temp"
+ else
+ echo "Leaving behind temp dirs in $temp"
+ fi
+}
+
+fatal() {
+ clear_temp
+ echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
+ exit 1
+}
+
+exit_cleanly() {
+ trap - INT
+ create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
+ rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+ stop_services
+ rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+ report_outcomes
+ clear_temp
+ exit ${#failures}
+}
+
+sanity_checks() {
+ ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ echo Checking dependencies:
+ echo -n 'virtualenv: '
+ virtualenv --version \
+ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
+ echo -n 'go: '
+ go version \
+ || fatal "No go binary. See http://golang.org/doc/install"
+ echo -n 'gcc: '
+ gcc --version | egrep ^gcc \
+ || fatal "No gcc. Try: apt-get install build-essential"
+ echo -n 'fuse.h: '
+ find /usr/include -wholename '*fuse/fuse.h' \
+ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'pyconfig.h: '
+ find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+ || fatal "No pyconfig.h. Try: apt-get install python-dev"
+ echo -n 'nginx: '
+ PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+ || fatal "No nginx. Try: apt-get install nginx"
+ echo -n 'perl: '
+ perl -v | grep version \
+ || fatal "No perl. Try: apt-get install perl"
+ for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
+ echo -n "perl $mod: "
+ perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
+ || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
+ done
+ echo -n 'gitolite: '
+ which gitolite \
+ || fatal "No gitolite. Try: apt-get install gitolite3"
+}
+
+rotate_logfile() {
+ # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+ # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
+ if [[ -f "$1/$2" ]]; then
+ THEDATE=`date +%Y%m%d%H%M%S`
+ mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
+ gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
+ fi
+}
+
+declare -a failures
+declare -A skip
+declare -A testargs
+skip[apps/workbench_profile]=1
+
+while [[ -n "$1" ]]
+do
+ arg="$1"; shift
+ case "$arg" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --skip)
+ skipwhat="$1"; shift
+ skip[$skipwhat]=1
+ ;;
+ --only)
+ only="$1"; skip[$1]=""; shift
+ ;;
+ --skip-install)
+ skip_install=1
+ ;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
+ --temp)
+ temp="$1"; shift
+ temp_preserve=1
+ ;;
+ --leave-temp)
+ temp_preserve=1
+ ;;
+ --retry)
+ retry=1
+ ;;
+ *_test=*)
+ suite="${arg%%_test=*}"
+ args="${arg#*=}"
+ testargs["$suite"]="$args"
+ ;;
+ *=*)
+ eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
+ ;;
+ *)
+ echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
+ exit 1
+ ;;
+ esac
+done
+
+start_api() {
+ echo 'Starting API server...'
+ cd "$WORKSPACE" \
+ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
+ && export ARVADOS_TEST_API_INSTALLED="$$" \
+ && (env | egrep ^ARVADOS)
+}
+
+start_nginx_proxy_services() {
+ echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && python sdk/python/tests/run_test_server.py start_keep-web \
+ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py start_nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1
+}
+
+stop_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ unset ARVADOS_TEST_PROXY_SERVICES
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy
+ fi
+ if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
+ unset ARVADOS_TEST_API_HOST
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop
+ fi
+}
+
+interrupt() {
+ failures+=("($(basename $0) interrupted)")
+ exit_cleanly
+}
+trap interrupt INT
+
+sanity_checks
+
+echo "WORKSPACE=$WORKSPACE"
+
+if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
+ # Jenkins expects us to use this by default.
+ CONFIGSRC="$HOME/arvados-api-server"
+fi
+
+# Clean up .pyc files that may exist in the workspace
+cd "$WORKSPACE"
+find -name '*.pyc' -delete
+
+if [[ -z "$temp" ]]; then
+ temp="$(mktemp -d)"
+fi
+
+# Set up temporary install dirs (unless existing dirs were supplied)
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+do
+ if [[ -z "${!tmpdir}" ]]; then
+ eval "$tmpdir"="$temp/$tmpdir"
+ fi
+ if ! [[ -d "${!tmpdir}" ]]; then
+ mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)"
+ fi
+done
+
+setup_ruby_environment() {
+ if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
+ source "$HOME/.rvm/scripts/rvm"
+ using_rvm=true
+ elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
+ source "/usr/local/rvm/scripts/rvm"
+ using_rvm=true
+ else
+ using_rvm=false
+ fi
+
+ if [[ "$using_rvm" == true ]]; then
+ # If rvm is in use, we can't just put separate "dependencies"
+ # and "gems-under-test" paths to GEM_PATH: passenger resets
+ # the environment to the "current gemset", which would lose
+ # our GEM_PATH and prevent our test suites from running ruby
+ # programs (for example, the Workbench test suite could not
+ # boot an API server or run arv). Instead, we have to make an
+ # rvm gemset and use it for everything.
+
+ [[ `type rvm | head -n1` == "rvm is a function" ]] \
+ || fatal 'rvm check'
+
+ # Put rvm's favorite path back in first place (overriding
+ # virtualenv, which just put itself there). Ignore rvm's
+ # complaint about not being in first place already.
+ rvm use @default 2>/dev/null
+
+ # Create (if needed) and switch to an @arvados-tests
+ # gemset. (Leave the choice of ruby to the caller.)
+ rvm use @arvados-tests --create \
+ || fatal 'rvm gemset setup'
+
+ rvm env
+ else
+ # When our "bundle install"s need to install new gems to
+ # satisfy dependencies, we want them to go where "gem install
+ # --user-install" would put them. (However, if the caller has
+ # already set GEM_HOME, we assume that's where dependencies
+ # should be installed, and we should leave it alone.)
+
+ if [ -z "$GEM_HOME" ]; then
+ user_gempath="$(gem env gempath)"
+ export GEM_HOME="${user_gempath%%:*}"
+ fi
+ PATH="$(gem env gemdir)/bin:$PATH"
+
+ # When we build and install our own gems, we install them in our
+ # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
+ # PATH so integration tests prefer them over other versions that
+ # happen to be installed in $user_gempath, system dirs, etc.
+
+ tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
+ PATH="$tmpdir_gem_home/bin:$PATH"
+ export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
+
+ echo "Will install dependencies to $(gem env gemdir)"
+ echo "Will install arvados gems to $tmpdir_gem_home"
+ echo "Gem search path is GEM_PATH=$GEM_PATH"
+ fi
+}
+
+with_test_gemset() {
+ if [[ "$using_rvm" == true ]]; then
+ "$@"
+ else
+ GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
+ fi
+}
+
+gem_uninstall_if_exists() {
+ if gem list "$1\$" | egrep '^\w'; then
+ gem uninstall --force --all --executables "$1"
+ fi
+}
+
+setup_virtualenv() {
+ local venvdest="$1"; shift
+ if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
+ virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+ fi
+ "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7'
+ # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
+ "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
+}
+
+export PERLINSTALLBASE
+export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+
+export GOPATH
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+ || fatal "symlink failed"
+
+setup_virtualenv "$VENVDIR" --python python2.7
+. "$VENVDIR/bin/activate"
+
+# Needed for run_test_server.py which is used by certain (non-Python) tests.
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+ || pip install PyYAML >/dev/null \
+ || fatal "pip install PyYAML failed"
+
+# Preinstall forked version of libcloud, because nodemanager "pip install"
+# won't pick it up by default.
+pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
+ || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
+ || fatal "pip install apache-libcloud failed"
+
+# This will help people who reuse --temp dirs when we upgrade to llfuse 0.42
+if egrep -q 'llfuse.*>= *0\.42' "$WORKSPACE/services/fuse/setup.py"; then
+ # Uninstall old llfuse, because services/fuse "pip install" won't
+ # upgrade it by default.
+ if pip freeze | egrep '^llfuse==0\.41\.'; then
+ yes | pip uninstall 'llfuse<0.42'
+ fi
+fi
+
+# Deactivate Python 2 virtualenv
+deactivate
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ setup_virtualenv "$VENV3DIR" --python python3
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
+
+# Reactivate Python 2 virtualenv
+. "$VENVDIR/bin/activate"
+
+# Note: this must be the last time we change PATH, otherwise rvm will
+# whine a lot.
+setup_ruby_environment
+
+echo "PATH is $PATH"
+
+if ! which bundler >/dev/null
+then
+ gem install --user-install bundler || fatal 'Could not install bundler'
+fi
+
+retry() {
+ while ! ${@} && [[ "$retry" == 1 ]]
+ do
+ read -p 'Try again? [Y/n] ' x
+ if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
+ then
+ break
+ fi
+ done
+}
+
+do_test() {
+ retry do_test_once ${@}
+}
+
+do_test_once() {
+ unset result
+ if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
+ then
+ title "Running $1 tests"
+ timer_reset
+ if [[ "$2" == "go" ]]
+ then
+ covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
+ coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ # We do "go get -t" here to catch compilation errors
+ # before trying "go test". Otherwise, coverage-reporting
+ # mode makes Go show the wrong line numbers when reporting
+ # compilation errors.
+ if [[ -n "${testargs[$1]}" ]]
+ then
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && \
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} ${testargs[$1]}
+ else
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+ fi
+ result="$?"
+ go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
+ rm "$WORKSPACE/tmp/.$covername.tmp"
+ elif [[ "$2" == "pip" ]]
+ then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
+ elif [[ "$2" != "" ]]
+ then
+ "test_$2"
+ else
+ "test_$1"
+ fi
+ result=${result:-$?}
+ checkexit $result "$1 tests"
+ title "End of $1 tests (`timer`)"
+ return $result
+ else
+ title "Skipping $1 tests"
+ fi
+}
+
+do_install() {
+ retry do_install_once ${@}
+}
+
+do_install_once() {
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
+ then
+ title "Running $1 install"
+ timer_reset
+ if [[ "$2" == "go" ]]
+ then
+ go get -t "git.curoverse.com/arvados.git/$1"
+ elif [[ "$2" == "pip" ]]
+ then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it believes the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually installed the local package
+ # we just built.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ elif [[ "$2" != "" ]]
+ then
+ "install_$2"
+ else
+ "install_$1"
+ fi
+ result=$?
+ checkexit $result "$1 install"
+ title "End of $1 install (`timer`)"
+ return $result
+ else
+ title "Skipping $1 install"
+ fi
+}
+
+bundle_install_trylocal() {
+ (
+ set -e
+ echo "(Running bundle install --local. 'could not find package' messages are OK.)"
+ if ! bundle install --local --no-deployment; then
+ echo "(Running bundle install again, without --local.)"
+ bundle install --no-deployment
+ fi
+ bundle package --all
+ )
+}
+
+install_doc() {
+ cd "$WORKSPACE/doc" \
+ && bundle_install_trylocal \
+ && rm -rf .site
+}
+do_install doc
+
+install_gem() {
+ gemname=$1
+ srcpath=$2
+ with_test_gemset gem_uninstall_if_exists "$gemname" \
+ && cd "$WORKSPACE/$srcpath" \
+ && bundle_install_trylocal \
+ && gem build "$gemname.gemspec" \
+ && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+}
+
+install_ruby_sdk() {
+ install_gem arvados sdk/ruby
+}
+do_install sdk/ruby ruby_sdk
+
+install_perl_sdk() {
+ cd "$WORKSPACE/sdk/perl" \
+ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
+ && make install INSTALLDIRS=perl
+}
+do_install sdk/perl perl_sdk
+
+install_cli() {
+ install_gem arvados-cli sdk/cli
+}
+do_install sdk/cli cli
+
+install_login-sync() {
+ install_gem arvados-login-sync services/login-sync
+}
+do_install services/login-sync login-sync
+
+# Install the Python SDK early. Various other test suites (like
+# keepproxy) bring up run_test_server.py, which imports the arvados
+# module. We can't actually *test* the Python SDK yet though, because
+# its own test suite brings up some of those other programs (like
+# keepproxy).
+declare -a pythonstuff
+pythonstuff=(
+ sdk/pam
+ sdk/python
+ sdk/cwl
+ services/fuse
+ services/nodemanager
+ tools/crunchstat-summary
+ )
+for p in "${pythonstuff[@]}"
+do
+ do_install "$p" pip
+done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
+
+install_apiserver() {
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle_install_trylocal
+
+ rm -f config/environments/test.rb
+ cp config/environments/test.rb.example config/environments/test.rb
+
+ if [ -n "$CONFIGSRC" ]
+ then
+ for f in database.yml application.yml
+ do
+ cp "$CONFIGSRC/$f" config/ || fatal "$f"
+ done
+ fi
+
+ # Fill in a random secret_token and blob_signing_key for testing
+ SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
+ BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
+
+ sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
+ sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
+
+ # Set up empty git repo (for git tests)
+ GITDIR=$(mktemp -d)
+ sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
+
+ rm -rf $GITDIR
+ mkdir -p $GITDIR/test
+ cd $GITDIR/test \
+ && git init \
+ && git config user.email "jenkins@ci.curoverse.com" \
+ && git config user.name "Jenkins, CI" \
+ && touch tmp \
+ && git add tmp \
+ && git commit -m 'initial commit'
+
+ # Clear out any lingering postgresql connections to the test
+ # database, so that we can drop it. This assumes the current user
+ # is a postgresql superuser.
+ cd "$WORKSPACE/services/api" \
+ && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle exec rake db:drop \
+ && RAILS_ENV=test bundle exec rake db:setup \
+ && RAILS_ENV=test bundle exec rake db:fixtures:load
+}
+do_install services/api apiserver
+
+declare -a gostuff
+gostuff=(
+ sdk/go/arvadosclient
+ sdk/go/blockdigest
+ sdk/go/manifest
+ sdk/go/streamer
+ sdk/go/crunchrunner
+ services/arv-git-httpd
+ services/crunchstat
+ services/keep-web
+ services/keepstore
+ sdk/go/keepclient
+ services/keepproxy
+ services/datamanager/summary
+ services/datamanager/collection
+ services/datamanager/keep
+ services/datamanager
+ services/crunch-dispatch-local
+ services/crunch-dispatch-slurm
+ services/crunch-run
+ tools/keep-rsync
+ )
+for g in "${gostuff[@]}"
+do
+ do_install "$g" go
+done
+
+install_workbench() {
+ cd "$WORKSPACE/apps/workbench" \
+ && mkdir -p tmp/cache \
+ && RAILS_ENV=test bundle_install_trylocal
+}
+do_install apps/workbench workbench
+
+test_doclinkchecker() {
+ (
+ set -e
+ cd "$WORKSPACE/doc"
+ ARVADOS_API_HOST=qr1hi.arvadosapi.com
+ # Make sure python-epydoc is installed or the next line won't
+ # do much good!
+ PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+ )
+}
+do_test doc doclinkchecker
+
+stop_services
+
+test_apiserver() {
+ rm -f "$WORKSPACE/services/api/git-commit.version"
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
+}
+do_test services/api apiserver
+
+# Shortcut for when we're only running apiserver tests. This saves a bit of time,
+# because we don't need to start up the api server for subsequent tests.
+if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
+ rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+ exit_cleanly
+fi
+
+start_api
+
+test_ruby_sdk() {
+ cd "$WORKSPACE/sdk/ruby" \
+ && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
+}
+do_test sdk/ruby ruby_sdk
+
+test_cli() {
+ cd "$WORKSPACE/sdk/cli" \
+ && mkdir -p /tmp/keep \
+ && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
+}
+do_test sdk/cli cli
+
+test_login-sync() {
+ cd "$WORKSPACE/services/login-sync" \
+ && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
+}
+do_test services/login-sync login-sync
+
+for p in "${pythonstuff[@]}"
+do
+ do_test "$p" pip
+done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
+
+for g in "${gostuff[@]}"
+do
+ do_test "$g" go
+done
+
+test_workbench() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
+}
+do_test apps/workbench workbench
+
+test_workbench_benchmark() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
+}
+do_test apps/workbench_benchmark workbench_benchmark
+
+test_workbench_profile() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
+}
+do_test apps/workbench_profile workbench_profile
+
+exit_cleanly
RUN apt-get update -q
## 20150915 nico -- fuse.postint has sporatic failures, spliting this up to see if it helps
RUN apt-get install -qy fuse
-RUN apt-get install -qy supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse crunchstat python-arvados-fuse cron dnsmasq
+RUN apt-get install -qy supervisor python-pip python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse crunchstat python-arvados-fuse cron dnsmasq
ADD fuse.conf /etc/fuse.conf
RUN chmod 644 /etc/fuse.conf
RUN apt-get update -q
RUN apt-get install -qy \
- python-pip python-pyvcf python-gflags python-google-api-python-client \
+ python-pip python-gflags python-google-api-python-client \
python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse \
crunchstat python-arvados-fuse cron vim supervisor openssh-server
# Determine whether this version of Docker supports memory+swap limits.
($exited, $stdout, $stderr) = srun_sync(
- ["srun", "--nodelist=" . $node[0]],
+ ["srun", "--nodes=1"],
[$docker_bin, 'run', '--help'],
{label => "check --memory-swap feature"});
$docker_limitmem = ($stdout =~ /--memory-swap/);
$try_user_arg = "--user=$try_user";
}
my ($exited, $stdout, $stderr) = srun_sync(
- ["srun", "--nodelist=" . $node[0]],
+ ["srun", "--nodes=1"],
["/bin/sh", "-ec",
"$docker_bin run $docker_run_args $try_user_arg $docker_hash id --user"],
{label => $label});
import cwltool.workflow
import cwltool.main
from cwltool.process import shortname
+from cwltool.errors import WorkflowException
import threading
import cwltool.docker
import fnmatch
if docker_req and kwargs.get("use_container") is not False:
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
+ resources = self.builder.resources
+ if resources is not None:
+ runtime_constraints["min_cores_per_node"] = resources.get("cores", 1)
+ runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
+ runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
+
try:
response = self.arvrunner.api.jobs().create(body={
"owner_uuid": self.arvrunner.project_uuid,
g = keepre.match(l)
if g:
keepdir = g.group(1)
- if tmpdir and outdir and keepdir:
- break
+
+ # It turns out if the job fails and restarts it can
+ # come up on a different compute node, so we have to
+ # read the log to the end to be sure instead of taking the
+ # easy way out.
+ #
+ #if tmpdir and outdir and keepdir:
+ # break
self.builder.outdir = outdir
self.builder.pathmapper.keepdir = keepdir
outputs = self.collect_outputs("keep:" + record["output"])
+ except WorkflowException as e:
+ logger.error("Error while collecting job outputs:\n%s", e, exc_info=(e if self.arvrunner.debug else False))
+ processStatus = "permanentFail"
except Exception as e:
- logger.exception("Got exception while collecting job outputs:")
+ logger.exception("Got unknown exception while collecting job outputs:")
processStatus = "permanentFail"
self.output_callback(outputs, processStatus)
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
+ self.debug = args.debug
+
try:
self.api.collections().get(uuid=crunchrunner_pdh).execute()
except arvados.errors.ApiError as e:
if sys.exc_info()[0] is KeyboardInterrupt:
logger.error("Interrupted, marking pipeline as failed")
else:
- logger.exception("Caught unhandled exception, marking pipeline as failed")
+ logger.error("Caught unhandled exception, marking pipeline as failed. Error was: %s", sys.exc_info()[0], exc_info=(sys.exc_info()[1] if self.debug else False))
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
finally:
'cwltool>=1.0.20160311170456',
'arvados-python-client>=0.1.20160219154918'
],
+ test_suite='tests',
+ tests_require=['mock>=1.0'],
zip_safe=True,
cmdclass={'egg_info': tagger},
)
--- /dev/null
+import unittest
+import mock
+import arvados_cwl
+
+class TestJob(unittest.TestCase):
+
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ def test_run(self):
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ tool = {
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls"
+ }
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
+ arvtool.formatgraph = None
+ for j in arvtool.job({}, "", mock.MagicMock()):
+ j.run()
+ runner.api.jobs().create.assert_called_with(body={
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'runtime_constraints': {},
+ 'script_parameters': {
+ 'tasks': [{
+ 'task.env': {'TMPDIR': '$(task.tmpdir)'},
+ 'command': ['ls']
+ }],
+ 'crunchrunner': '83db29f08544e1c319572a6bd971088a+140/crunchrunner'
+ },
+ 'script_version': 'master',
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script': 'crunchrunner',
+ 'runtime_constraints': {
+ 'min_cores_per_node': 1,
+ 'min_ram_mb_per_node': 1024,
+ 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
+ }
+ }, find_or_create=True)
+
+ # The test passes some fields in builder.resources
+ # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ def test_resource_requirements(self):
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ tool = {
+ "inputs": [],
+ "outputs": [],
+ "hints": [{
+ "class": "ResourceRequirement",
+ "coresMin": 3,
+ "ramMin": 3000,
+ "tmpdirMin": 4000
+ }],
+ "baseCommand": "ls"
+ }
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
+ arvtool.formatgraph = None
+ for j in arvtool.job({}, "", mock.MagicMock()):
+ j.run()
+ runner.api.jobs().create.assert_called_with(body={
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'runtime_constraints': {},
+ 'script_parameters': {
+ 'tasks': [{
+ 'task.env': {'TMPDIR': '$(task.tmpdir)'},
+ 'command': ['ls']
+ }],
+ 'crunchrunner': '83db29f08544e1c319572a6bd971088a+140/crunchrunner'
+ },
+ 'script_version': 'master',
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script': 'crunchrunner',
+ 'runtime_constraints': {
+ 'min_cores_per_node': 3,
+ 'min_ram_mb_per_node': 3000,
+ 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
+ }
+ }, find_or_create=True)
// error.
type ResponseWriter struct {
http.ResponseWriter
- wroteStatus *int // Last status given to WriteHeader()
- wroteBodyBytes *int // Bytes successfully written
- err *error // Last error returned from Write()
+ wroteStatus *int // Last status given to WriteHeader()
+ wroteBodyBytes *int // Bytes successfully written
+ err *error // Last error returned from Write()
}
func WrapResponseWriter(orig http.ResponseWriter) ResponseWriter {
cache_pos, cache_data = self._readline_cache
if self.tell() == cache_pos:
data = [cache_data]
+ self._filepos += len(cache_data)
else:
data = ['']
data_size = len(data[-1])
except ValueError:
nextline_index = len(data)
nextline_index = min(nextline_index, size)
+ self._filepos -= len(data) - nextline_index
self._readline_cache = (self.tell(), data[nextline_index:])
return data[:nextline_index]
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
+ @unittest.skip("unreliable test - please fix and close #8752")
def test_KeepSingleCopyRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
blob_locator = self.keep_client.put(blob_str, copies=1)
def test_bz2_decompression(self):
self.check_decompression('bz2', bz2.compress)
+ def test_readline_then_readlines(self):
+ reader = self.make_newlines_reader()
+ data = reader.readline()
+ self.assertEqual('one\n', data)
+ data = reader.readlines()
+ self.assertEqual(['two\n', '\n', 'three\n', 'four\n', '\n'], data)
+
+ def test_readline_then_readall(self):
+ reader = self.make_newlines_reader()
+ data = reader.readline()
+ self.assertEqual('one\n', data)
+ self.assertEqual(''.join(['two\n', '\n', 'three\n', 'four\n', '\n']), ''.join(reader.readall()))
+
class StreamRetryTestMixin(object):
# Define reader_for(coll_name, **kwargs)
"lib/arvados/collection.rb", "lib/arvados/keep.rb",
"README", "LICENSE-2.0.txt"]
s.required_ruby_version = '>= 2.1.0'
+ # activesupport <4.2.6 only because https://dev.arvados.org/issues/8222
+ s.add_dependency('activesupport', '>= 3.2.13', '< 4.2.6')
+ s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
s.add_dependency('google-api-client', '~> 0.6.3', '>= 0.6.3')
- s.add_dependency('activesupport', '>= 3.2.13')
s.add_dependency('json', '~> 1.7', '>= 1.7.7')
- s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
s.add_runtime_dependency('jwt', '>= 0.1.5', '< 1.0.0')
s.homepage =
'https://arvados.org'
def find_objects_for_index
# Here we are deliberately less helpful about searching for client
# authorizations. We look up tokens belonging to the current user
- # and filter by exact matches on api_token and scopes.
+ # and filter by exact matches on uuid, api_token, and scopes.
wanted_scopes = []
if @filters
wanted_scopes.concat(@filters.map { |attr, operator, operand|
((attr == 'scopes') and (operator == '=')) ? operand : nil
})
@filters.select! { |attr, operator, operand|
- ((attr == 'uuid') and (operator == '=')) || ((attr == 'api_token') and (operator == '='))
+ operator == '=' && (attr == 'uuid' || attr == 'api_token')
}
end
if @where
wanted_scopes << @where['scopes']
- @where.select! { |attr, val| attr == 'uuid' }
+ @where.select! { |attr, val|
+ # "where":{"uuid":"zzzzz-zzzzz-zzzzzzzzzzzzzzz"} is OK but
+ # "where":{"api_client_id":1} is not supported
+ # "where":{"uuid":["contains","-"]} is not supported
+ # "where":{"uuid":["uuid1","uuid2","uuid3"]} is not supported
+ val.is_a?(String) && (attr == 'uuid' || attr == 'api_token')
+ }
end
@objects = model_class.
includes(:user, :api_client).
end
def find_object_by_uuid
- @object = model_class.where(uuid: (params[:uuid] || params[:id])).first
+ uuid_param = params[:uuid] || params[:id]
+ if (uuid_param != current_api_client_authorization.andand.uuid and
+ not Thread.current[:api_client].andand.is_trusted)
+ return forbidden
+ end
+ @limit = 1
+ @offset = 0
+ @orders = []
+ @where = {}
+ @filters = [['uuid', '=', uuid_param]]
+ find_objects_for_index
+ @object = @objects.first
end
def current_api_client_is_trusted
- unless Thread.current[:api_client].andand.is_trusted
- if params["action"] == "show"
- if @object and @object['api_token'] == current_api_client_authorization.andand.api_token
- return true
- end
- elsif params["action"] == "index" and @objects.andand.size == 1
- filters = @filters.map{|f|f.first}.uniq
- if ['uuid'] == filters
- return true if @objects.first['api_token'] == current_api_client_authorization.andand.api_token
- elsif ['api_token'] == filters
- return true if @objects.first[:user_id] = current_user.id
- end
- end
- send_error('Forbidden: this API client cannot manipulate other clients\' access tokens.',
- status: 403)
+ if Thread.current[:api_client].andand.is_trusted
+ return true
+ end
+ # A non-trusted client can do a search for its own token if it
+ # explicitly restricts the search to its own UUID or api_token.
+ # Any other kind of query must return 403, even if it matches only
+ # the current token, because that's currently how Workbench knows
+ # (after searching on scopes) the difference between "the token
+ # I'm using now *is* the only sharing token for this collection"
+ # (403) and "my token is trusted, and there is one sharing token
+ # for this collection" (200).
+ #
+ # The @filters test here also prevents a non-trusted token from
+ # filtering on its own scopes, and discovering whether any _other_
+ # equally scoped tokens exist (403=yes, 200=no).
+ if (@objects.andand.count == 1 and
+ @objects.first.uuid == current_api_client_authorization.andand.uuid and
+ (@filters.map(&:first) & %w(uuid api_token)).any?)
+ return true
end
+ forbidden
+ end
+
+ def forbidden
+ send_error('Forbidden: this API client cannot manipulate other clients\' access tokens.',
+ status: 403)
end
end
def permission_to_update
(permission_to_create and
- not self.user_id_changed? and
- not self.owner_uuid_changed?)
+ not uuid_changed? and
+ not user_id_changed? and
+ not owner_uuid_changed?)
end
def log_update
end
end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 403], # this belongs to the user of current session, but we can't get it by uuid
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} get #{token} token and expect #{status}" do
+ [# anyone can look up the token they're currently using
+ [:admin, :admin, 200, 200, 1],
+ [:active, :active, 200, 200, 1],
+ # cannot look up other tokens (even for same user) if not trustedclient
+ [:admin, :active, 403, 403],
+ [:admin, :admin_vm, 403, 403],
+ [:active, :admin, 403, 403],
+ # cannot look up other tokens for other users, regardless of trustedclient
+ [:admin_trustedclient, :active, 404, 200, 0],
+ [:active_trustedclient, :admin, 404, 200, 0],
+ ].each do |user, token, expect_get_response, expect_list_response, expect_list_items|
+ test "using '#{user}', get '#{token}' by uuid" do
authorize_with user
- get :show, {id: api_client_authorizations(token).uuid}
- assert_response status
+ get :show, {
+ id: api_client_authorizations(token).uuid,
+ }
+ assert_response expect_get_response
+ end
+
+ test "using '#{user}', update '#{token}' by uuid" do
+ authorize_with user
+ put :update, {
+ id: api_client_authorizations(token).uuid,
+ api_client_authorization: {},
+ }
+ assert_response expect_get_response
+ end
+
+ test "using '#{user}', delete '#{token}' by uuid" do
+ authorize_with user
+ post :destroy, {
+ id: api_client_authorizations(token).uuid,
+ }
+ assert_response expect_get_response
end
- end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 403], # this belongs to the user of current session, but we can't list it by uuid
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} list #{token} token using uuid and expect #{status}" do
+ test "using '#{user}', list '#{token}' by uuid" do
authorize_with user
get :index, {
- filters: [['uuid','=',api_client_authorizations(token).uuid]]
+ filters: [['uuid','=',api_client_authorizations(token).uuid]],
}
- assert_response status
+ assert_response expect_list_response
+ if expect_list_items
+ assert_equal assigns(:objects).length, expect_list_items
+ end
end
- end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 200], # this belongs to the user of current session, and can be listed by token
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} list #{token} token using token and expect #{status}" do
+ test "using '#{user}', list '#{token}' by token" do
authorize_with user
get :index, {
- filters: [['api_token','=',api_client_authorizations(token).api_token]]
+ filters: [['api_token','=',api_client_authorizations(token).api_token]],
}
- assert_response status
+ assert_response expect_list_response
+ if expect_list_items
+ assert_equal assigns(:objects).length, expect_list_items
+ end
end
end
+
+ test "scoped token cannot change its own scopes" do
+ authorize_with :admin_vm
+ put :update, {
+ id: api_client_authorizations(:admin_vm).uuid,
+ api_client_authorization: {scopes: ['all']},
+ }
+ assert_response 403
+ end
+
+ test "token cannot change its own uuid" do
+ authorize_with :admin
+ put :update, {
+ id: api_client_authorizations(:admin).uuid,
+ api_client_authorization: {uuid: 'zzzzz-gj3su-zzzzzzzzzzzzzzz'},
+ }
+ assert_response 403
+ end
end
end
[
- {cfg: :git_repo_ssh_base, cfgval: "git@example.com:", match: %r"^git@example.com:/"},
- {cfg: :git_repo_ssh_base, cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:/"},
+ {cfg: :git_repo_ssh_base, cfgval: "git@example.com:", match: %r"^git@example.com:"},
+ {cfg: :git_repo_ssh_base, cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:"},
{cfg: :git_repo_ssh_base, cfgval: false, refute: /^git@/ },
- {cfg: :git_repo_https_base, cfgval: "https://example.com/", match: %r"https://example.com/"},
+ {cfg: :git_repo_https_base, cfgval: "https://example.com/", match: %r"^https://example.com/"},
{cfg: :git_repo_https_base, cfgval: true, match: %r"^https://git.zzzzz.arvadosapi.com/"},
{cfg: :git_repo_https_base, cfgval: false, refute: /^http/ },
].each do |expect|
authorize_with :active
get :index
assert_response :success
+ assert_not_empty json_response['items']
json_response['items'].each do |r|
if expect[:refute]
r['clone_urls'].each do |u|
refute_match expect[:refute], u
end
else
- assert r['clone_urls'].any? do |u|
- expect[:prefix].match u
- end
+ assert((r['clone_urls'].any? do |u|
+ expect[:match].match u
+ end),
+ "no match for #{expect[:match]} in #{r['clone_urls'].inspect}")
end
end
end
--- /dev/null
+package main
+
+import (
+ "flag"
+ "fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "os/signal"
+ "sync"
+ "syscall"
+ "time"
+)
+
+func main() {
+ err := doMain()
+ if err != nil {
+ log.Fatalf("%q", err)
+ }
+}
+
+var (
+ arv arvadosclient.ArvadosClient
+ runningCmds map[string]*exec.Cmd
+ runningCmdsMutex sync.Mutex
+ waitGroup sync.WaitGroup
+ doneProcessing chan bool
+ sigChan chan os.Signal
+)
+
+func doMain() error {
+ flags := flag.NewFlagSet("crunch-dispatch-slurm", flag.ExitOnError)
+
+ pollInterval := flags.Int(
+ "poll-interval",
+ 10,
+ "Interval in seconds to poll for queued containers")
+
+ priorityPollInterval := flags.Int(
+ "container-priority-poll-interval",
+ 60,
+ "Interval in seconds to check priority of a dispatched container")
+
+ crunchRunCommand := flags.String(
+ "crunch-run-command",
+ "/usr/bin/crunch-run",
+ "Crunch command to run container")
+
+ finishCommand := flags.String(
+ "finish-command",
+ "/usr/bin/crunch-finish-slurm.sh",
+ "Command to run from strigger when job is finished")
+
+ // Parse args; omit the first arg which is the command name
+ flags.Parse(os.Args[1:])
+
+ var err error
+ arv, err = arvadosclient.MakeArvadosClient()
+ if err != nil {
+ return err
+ }
+
+ // Channel to terminate
+ doneProcessing = make(chan bool)
+
+ // Graceful shutdown
+ sigChan = make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+ go func(sig <-chan os.Signal) {
+ for sig := range sig {
+ log.Printf("Caught signal: %v", sig)
+ doneProcessing <- true
+ }
+ }(sigChan)
+
+ // Run all queued containers
+ runQueuedContainers(*pollInterval, *priorityPollInterval, *crunchRunCommand, *finishCommand)
+
+ // Wait for all running crunch jobs to complete / terminate
+ waitGroup.Wait()
+
+ return nil
+}
+
+// Poll for queued containers using pollInterval.
+// Invoke dispatchSlurm for each ticker cycle, which will run all the queued containers.
+//
+// Any errors encountered are logged but the program would continue to run (not exit).
+// This is because, once one or more crunch jobs are running,
+// we would need to wait for them complete.
+func runQueuedContainers(pollInterval, priorityPollInterval int, crunchRunCommand, finishCommand string) {
+ ticker := time.NewTicker(time.Duration(pollInterval) * time.Second)
+
+ for {
+ select {
+ case <-ticker.C:
+ dispatchSlurm(priorityPollInterval, crunchRunCommand, finishCommand)
+ case <-doneProcessing:
+ ticker.Stop()
+ return
+ }
+ }
+}
+
+// Container data
+type Container struct {
+ UUID string `json:"uuid"`
+ State string `json:"state"`
+ Priority int `json:"priority"`
+}
+
+// ContainerList is a list of the containers from api
+type ContainerList struct {
+ Items []Container `json:"items"`
+}
+
+// Get the list of queued containers from API server and invoke run for each container.
+func dispatchSlurm(priorityPollInterval int, crunchRunCommand, finishCommand string) {
+ params := arvadosclient.Dict{
+ "filters": [][]string{[]string{"state", "=", "Queued"}},
+ }
+
+ var containers ContainerList
+ err := arv.List("containers", params, &containers)
+ if err != nil {
+ log.Printf("Error getting list of queued containers: %q", err)
+ return
+ }
+
+ for i := 0; i < len(containers.Items); i++ {
+ log.Printf("About to submit queued container %v", containers.Items[i].UUID)
+ // Run the container
+ go run(containers.Items[i], crunchRunCommand, finishCommand, priorityPollInterval)
+ }
+}
+
+// sbatchCmd
+func sbatchFunc(uuid string) *exec.Cmd {
+ return exec.Command("sbatch", "--job-name="+uuid, "--share", "--parsable")
+}
+
+var sbatchCmd = sbatchFunc
+
+// striggerCmd
+func striggerFunc(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) *exec.Cmd {
+ return exec.Command("strigger", "--set", "--jobid="+jobid, "--fini",
+ fmt.Sprintf("--program=%s %s %s %s %s", finishCommand, apiHost, apiToken, apiInsecure, containerUUID))
+}
+
+var striggerCmd = striggerFunc
+
+// Submit job to slurm using sbatch.
+func submit(container Container, crunchRunCommand string) (jobid string, submitErr error) {
+ submitErr = nil
+
+ // Mark record as complete if anything errors out.
+ defer func() {
+ if submitErr != nil {
+ // This really should be an "Error" state, see #8018
+ updateErr := arv.Update("containers", container.UUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Complete"}},
+ nil)
+ if updateErr != nil {
+ log.Printf("Error updating container state to 'Complete' for %v: %q", container.UUID, updateErr)
+ }
+ }
+ }()
+
+ // Create the command and attach to stdin/stdout
+ cmd := sbatchCmd(container.UUID)
+ stdinWriter, stdinerr := cmd.StdinPipe()
+ if stdinerr != nil {
+ submitErr = fmt.Errorf("Error creating stdin pipe %v: %q", container.UUID, stdinerr)
+ return
+ }
+
+ stdoutReader, stdoutErr := cmd.StdoutPipe()
+ if stdoutErr != nil {
+ submitErr = fmt.Errorf("Error creating stdout pipe %v: %q", container.UUID, stdoutErr)
+ return
+ }
+
+ stderrReader, stderrErr := cmd.StderrPipe()
+ if stderrErr != nil {
+ submitErr = fmt.Errorf("Error creating stderr pipe %v: %q", container.UUID, stderrErr)
+ return
+ }
+
+ err := cmd.Start()
+ if err != nil {
+ submitErr = fmt.Errorf("Error starting %v: %v", cmd.Args, err)
+ return
+ }
+
+ stdoutChan := make(chan []byte)
+ go func() {
+ b, _ := ioutil.ReadAll(stdoutReader)
+ stdoutChan <- b
+ close(stdoutChan)
+ }()
+
+ stderrChan := make(chan []byte)
+ go func() {
+ b, _ := ioutil.ReadAll(stderrReader)
+ stderrChan <- b
+ close(stderrChan)
+ }()
+
+ // Send a tiny script on stdin to execute the crunch-run command
+ // slurm actually enforces that this must be a #! script
+ fmt.Fprintf(stdinWriter, "#!/bin/sh\nexec '%s' '%s'\n", crunchRunCommand, container.UUID)
+ stdinWriter.Close()
+
+ err = cmd.Wait()
+
+ stdoutMsg := <-stdoutChan
+ stderrmsg := <-stderrChan
+
+ if err != nil {
+ submitErr = fmt.Errorf("Container submission failed %v: %v %v", cmd.Args, err, stderrmsg)
+ return
+ }
+
+ // If everything worked out, got the jobid on stdout
+ jobid = string(stdoutMsg)
+
+ return
+}
+
+// finalizeRecordOnFinish uses 'strigger' command to register a script that will run on
+// the slurm controller when the job finishes.
+func finalizeRecordOnFinish(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) {
+ cmd := striggerCmd(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure)
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ log.Printf("While setting up strigger: %v", err)
+ }
+}
+
+// Run a queued container.
+// Set container state to locked (TBD)
+// Submit job to slurm to execute crunch-run command for the container
+// If the container priority becomes zero while crunch job is still running, cancel the job.
+func run(container Container, crunchRunCommand, finishCommand string, priorityPollInterval int) {
+
+ jobid, err := submit(container, crunchRunCommand)
+ if err != nil {
+ log.Printf("Error queuing container run: %v", err)
+ return
+ }
+
+ insecure := "0"
+ if arv.ApiInsecure {
+ insecure = "1"
+ }
+ finalizeRecordOnFinish(jobid, container.UUID, finishCommand, arv.ApiServer, arv.ApiToken, insecure)
+
+ // Update container status to Running, this is a temporary workaround
+ // to avoid resubmitting queued containers because record locking isn't
+ // implemented yet.
+ err = arv.Update("containers", container.UUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Running"}},
+ nil)
+ if err != nil {
+ log.Printf("Error updating container state to 'Running' for %v: %q", container.UUID, err)
+ }
+
+ log.Printf("Submitted container run for %v", container.UUID)
+
+ containerUUID := container.UUID
+
+ // A goroutine to terminate the runner if container priority becomes zero
+ priorityTicker := time.NewTicker(time.Duration(priorityPollInterval) * time.Second)
+ go func() {
+ for _ = range priorityTicker.C {
+ var container Container
+ err := arv.Get("containers", containerUUID, nil, &container)
+ if err != nil {
+ log.Printf("Error getting container info for %v: %q", container.UUID, err)
+ } else {
+ if container.Priority == 0 {
+ log.Printf("Canceling container %v", container.UUID)
+ priorityTicker.Stop()
+ cancelcmd := exec.Command("scancel", "--name="+container.UUID)
+ cancelcmd.Run()
+ }
+ if container.State == "Complete" {
+ priorityTicker.Stop()
+ }
+ }
+ }
+ }()
+
+}
--- /dev/null
+package main
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+
+ "io/ioutil"
+ "log"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+var _ = Suite(&TestSuite{})
+var _ = Suite(&MockArvadosServerSuite{})
+
+type TestSuite struct{}
+type MockArvadosServerSuite struct{}
+
+var initialArgs []string
+
+func (s *TestSuite) SetUpSuite(c *C) {
+ initialArgs = os.Args
+ arvadostest.StartAPI()
+}
+
+func (s *TestSuite) TearDownSuite(c *C) {
+ arvadostest.StopAPI()
+}
+
+func (s *TestSuite) SetUpTest(c *C) {
+ args := []string{"crunch-dispatch-slurm"}
+ os.Args = args
+
+ var err error
+ arv, err = arvadosclient.MakeArvadosClient()
+ if err != nil {
+ c.Fatalf("Error making arvados client: %s", err)
+ }
+}
+
+func (s *TestSuite) TearDownTest(c *C) {
+ arvadostest.ResetEnv()
+ os.Args = initialArgs
+}
+
+func (s *MockArvadosServerSuite) TearDownTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *TestSuite) Test_doMain(c *C) {
+ args := []string{"-poll-interval", "2", "-container-priority-poll-interval", "1", "-crunch-run-command", "echo"}
+ os.Args = append(os.Args, args...)
+
+ var sbatchCmdLine []string
+ var striggerCmdLine []string
+
+ // Override sbatchCmd
+ defer func(orig func(string) *exec.Cmd) {
+ sbatchCmd = orig
+ }(sbatchCmd)
+ sbatchCmd = func(uuid string) *exec.Cmd {
+ sbatchCmdLine = sbatchFunc(uuid).Args
+ return exec.Command("echo", uuid)
+ }
+
+ // Override striggerCmd
+ defer func(orig func(jobid, containerUUID, finishCommand,
+ apiHost, apiToken, apiInsecure string) *exec.Cmd) {
+ striggerCmd = orig
+ }(striggerCmd)
+ striggerCmd = func(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) *exec.Cmd {
+ striggerCmdLine = striggerFunc(jobid, containerUUID, finishCommand,
+ apiHost, apiToken, apiInsecure).Args
+ go func() {
+ time.Sleep(5 * time.Second)
+ arv.Update("containers", containerUUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Complete"}},
+ nil)
+ }()
+ return exec.Command("echo", "strigger")
+ }
+
+ go func() {
+ time.Sleep(8 * time.Second)
+ sigChan <- syscall.SIGINT
+ }()
+
+ // There should be no queued containers now
+ params := arvadosclient.Dict{
+ "filters": [][]string{[]string{"state", "=", "Queued"}},
+ }
+ var containers ContainerList
+ err := arv.List("containers", params, &containers)
+ c.Check(err, IsNil)
+ c.Check(len(containers.Items), Equals, 1)
+
+ err = doMain()
+ c.Check(err, IsNil)
+
+ c.Check(sbatchCmdLine, DeepEquals, []string{"sbatch", "--job-name=zzzzz-dz642-queuedcontainer", "--share", "--parsable"})
+ c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini",
+ "--program=/usr/bin/crunch-finish-slurm.sh " + os.Getenv("ARVADOS_API_HOST") + " 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h 1 zzzzz-dz642-queuedcontainer"})
+
+ // There should be no queued containers now
+ err = arv.List("containers", params, &containers)
+ c.Check(err, IsNil)
+ c.Check(len(containers.Items), Equals, 0)
+
+ // Previously "Queued" container should now be in "Complete" state
+ var container Container
+ err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container)
+ c.Check(err, IsNil)
+ c.Check(container.State, Equals, "Complete")
+}
+
+func (s *MockArvadosServerSuite) Test_APIErrorGettingContainers(c *C) {
+ apiStubResponses := make(map[string]arvadostest.StubResponse)
+ apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)}
+
+ testWithServerStub(c, apiStubResponses, "echo", "Error getting list of queued containers")
+}
+
+func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) {
+ apiStub := arvadostest.ServerStub{apiStubResponses}
+
+ api := httptest.NewServer(&apiStub)
+ defer api.Close()
+
+ arv = arvadosclient.ArvadosClient{
+ Scheme: "http",
+ ApiServer: api.URL[7:],
+ ApiToken: "abc123",
+ Client: &http.Client{Transport: &http.Transport{}},
+ Retries: 0,
+ }
+
+ tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
+ c.Check(err, IsNil)
+ defer os.Remove(tempfile.Name())
+ log.SetOutput(tempfile)
+
+ go func() {
+ time.Sleep(2 * time.Second)
+ sigChan <- syscall.SIGTERM
+ }()
+
+ runQueuedContainers(2, 1, crunchCmd, crunchCmd)
+
+ buf, _ := ioutil.ReadFile(tempfile.Name())
+ c.Check(strings.Contains(string(buf), expected), Equals, true)
+}
--- /dev/null
+#!/bin/sh
+
+# Script to be called by strigger when a job finishes. This ensures the job
+# record has the correct state "Complete" even if the node running the job
+# failed.
+
+ARVADOS_API_HOST=$1
+ARVADOS_API_TOKEN=$2
+ARVADOS_API_HOST_INSECURE=$3
+uuid=$4
+jobid=$5
+
+# If it is possible to attach metadata to job records we could look up the
+# above information instead of getting it on the command line. For example,
+# this is the recipe for getting the job name (container uuid) from the job id.
+#uuid=$(squeue --jobs=$jobid --states=all --format=%j --noheader)
+
+export ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE
+
+exec arv container update --uuid $uuid --container '{"state": "Complete"}'
"select": fieldsWanted,
"order": []string{"modified_at ASC", "uuid ASC"},
"filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}},
- "offset": 0}
+ "offset": 0}
if params.BatchSize > 0 {
sdkParams["limit"] = params.BatchSize
}
if totalCollections < finalNumberOfCollectionsAvailable {
err = fmt.Errorf("API server indicates a total of %d collections "+
- "available up to %v, but we only retrieved %d. "+
- "Refusing to continue as this could indicate an "+
- "otherwise undetected failure.",
- finalNumberOfCollectionsAvailable,
- sdkParams["filters"].([][]string)[0][2],
- totalCollections)
+ "available up to %v, but we only retrieved %d. "+
+ "Refusing to continue as this could indicate an "+
+ "otherwise undetected failure.",
+ finalNumberOfCollectionsAvailable,
+ sdkParams["filters"].([][]string)[0][2],
+ totalCollections)
return
}
def _remove_container(self, cid):
try:
- self.docker_client.remove_container(cid)
+ self.docker_client.remove_container(cid, v=True)
except docker.errors.APIError as error:
logger.warning("Failed to remove container %s: %s", cid, error)
else:
TEST_CLASS = cleaner.DockerImageCleaner
TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True}
+ def test_container_deletion_deletes_volumes(self):
+ cid = MockDockerId()
+ self.events.append(MockEvent('die', docker_id=cid))
+ self.recorder.run()
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
+
@mock.patch('arvados_docker.cleaner.logger')
def test_failed_container_deletion_handling(self, mockLogger):
cid = MockDockerId()
self.docker_client.remove_container.side_effect = MockException(500)
self.events.append(MockEvent('die', docker_id=cid))
self.recorder.run()
- self.docker_client.remove_container.assert_called_with(cid)
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
self.assertEqual("Failed to remove container %s: %s",
mockLogger.warning.call_args[0][0])
self.assertEqual(cid,
def test_remove_onexit(self):
self.args.remove_stopped_containers = 'onexit'
cleaner.run(self.args, self.docker_client)
- self.docker_client.remove_container.assert_called_once_with(self.newCID)
+ self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
def test_remove_always(self):
self.args.remove_stopped_containers = 'always'
cleaner.run(self.args, self.docker_client)
- self.docker_client.remove_container.assert_any_call(self.existingCID)
- self.docker_client.remove_container.assert_any_call(self.newCID)
+ self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
+ self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
mock.call.events(since=mock.ANY),
mock.call.containers(filters={'status':'exited'})])
# Asked to delete the container twice?
- self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID)] * 2)
+ self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID, v=True)] * 2)
self.assertEqual(2, self.docker_client.remove_container.call_count)
func (v *AzureBlobVolume) isKeepBlock(s string) bool {
return keepBlockRegexp.MatchString(s)
}
+
+// EmptyTrash looks for trashed blocks that exceeded trashLifetime
+// and deletes them from the volume.
+// TBD
+func (v *AzureBlobVolume) EmptyTrash() {
+}
// trashLifetime is the time duration after a block is trashed
// during which it can be recovered using an /untrash request
+// Use 10s or 10m or 10h to set as 10 seconds or minutes or hours respectively.
var trashLifetime time.Duration
+// trashCheckInterval is the time duration at which the emptyTrash goroutine
+// will check and delete expired trashed blocks. Default is one day.
+// Use 10s or 10m or 10h to set as 10 seconds or minutes or hours respectively.
+var trashCheckInterval time.Duration
+
var maxBuffers = 128
var bufs *bufferPool
&trashLifetime,
"trash-lifetime",
0*time.Second,
- "Interval after a block is trashed during which it can be recovered using an /untrash request")
+ "Time duration after a block is trashed during which it can be recovered using an /untrash request")
+ flag.DurationVar(
+ &trashCheckInterval,
+ "trash-check-interval",
+ 24*time.Hour,
+ "Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day.")
flag.Parse()
trashq = NewWorkQueue()
go RunTrashWorker(trashq)
+ // Start emptyTrash goroutine
+ doneEmptyingTrash := make(chan bool)
+ go emptyTrash(doneEmptyingTrash, trashCheckInterval)
+
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
term := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
s := <-sig
log.Println("caught signal:", s)
+ doneEmptyingTrash <- true
listener.Close()
}(term)
signal.Notify(term, syscall.SIGTERM)
srv := &http.Server{Addr: listen}
srv.Serve(listener)
}
+
+// At every trashCheckInterval tick, invoke EmptyTrash on all volumes.
+func emptyTrash(doneEmptyingTrash chan bool, trashCheckInterval time.Duration) {
+ ticker := time.NewTicker(trashCheckInterval)
+
+ for {
+ select {
+ case <-ticker.C:
+ for _, v := range volumes {
+ if v.Writable() {
+ v.EmptyTrash()
+ }
+ }
+ case <-doneEmptyingTrash:
+ ticker.Stop()
+ return
+ }
+ }
+}
return nil
}
+// Trash a Keep block.
func (v *S3Volume) Trash(loc string) error {
if v.readonly {
return MethodDisabledError
}
return err
}
+
+// EmptyTrash looks for trashed blocks that exceeded trashLifetime
+// and deletes them from the volume.
+// TBD
+func (v *S3Volume) EmptyTrash() {
+}
// underlying device. It will be passed on to clients in
// responses to PUT requests.
Replication() int
+
+ // EmptyTrash looks for trashed blocks that exceeded trashLifetime
+ // and deletes them from the volume.
+ EmptyTrash()
}
// A VolumeManager tells callers which volumes can read, which volumes
testPutFullBlock(t, factory)
testTrashUntrash(t, factory)
+ testTrashEmptyTrashUntrash(t, factory)
}
// Put a test block, get it and verify content
v := factory(t)
defer v.Teardown()
defer func() {
- trashLifetime = 0
+ trashLifetime = 0 * time.Second
}()
trashLifetime = 3600 * time.Second
}
bufs.Put(buf)
}
+
+func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
+ v := factory(t)
+ defer v.Teardown()
+ defer func(orig time.Duration) {
+ trashLifetime = orig
+ }(trashLifetime)
+
+ checkGet := func() error {
+ buf, err := v.Get(TestHash)
+ if err != nil {
+ return err
+ }
+ if bytes.Compare(buf, TestBlock) != 0 {
+ t.Fatalf("Got data %+q, expected %+q", buf, TestBlock)
+ }
+ bufs.Put(buf)
+ return nil
+ }
+
+ // First set: EmptyTrash before reaching the trash deadline.
+
+ trashLifetime = 1 * time.Hour
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ err := checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = v.Trash(TestHash)
+ if err == MethodDisabledError || err == ErrNotImplemented {
+ // Skip the trash tests for read-only volumes, and
+ // volume types that don't support trashLifetime>0.
+ return
+ }
+
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ v.EmptyTrash()
+
+ // Even after emptying the trash, we can untrash our block
+ // because the deadline hasn't been reached.
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Untrash should fail if the only block in the trash has
+ // already been untrashed.
+ err = v.Untrash(TestHash)
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // The failed Untrash should not interfere with our
+ // already-untrashed copy.
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Second set: EmptyTrash after the trash deadline has passed.
+
+ trashLifetime = 1 * time.Nanosecond
+
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Even though 1ns has passed, we can untrash because we
+ // haven't called EmptyTrash yet.
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Trash it again, and this time call EmptyTrash so it really
+ // goes away.
+ err = v.Trash(TestHash)
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Errorf("os.IsNotExist(%v) should have been true", err)
+ }
+ v.EmptyTrash()
+
+ // Untrash won't find it
+ err = v.Untrash(TestHash)
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Get block won't find it
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Third set: If the same data block gets written again after
+ // being trashed, and then the trash gets emptied, the newer
+ // un-trashed copy doesn't get deleted along with it.
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Nanosecond
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ // EmptyTrash should not delete the untrashed copy.
+ v.EmptyTrash()
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Fourth set: If the same data block gets trashed twice with
+ // different deadlines A and C, and then the trash is emptied
+ // at intermediate time B (A < B < C), it is still possible to
+ // untrash the block whose deadline is "C".
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Nanosecond
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Hour
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // EmptyTrash should not prevent us from recovering the
+ // time.Hour ("C") trash
+ v.EmptyTrash()
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+}
func (v *MockVolume) Replication() int {
return 1
}
+
+func (v *MockVolume) EmptyTrash() {
+}
}
func (vs *unixVolumeAdder) Set(value string) error {
- if trashLifetime != 0 {
- return ErrNotImplemented
- }
if dirs := strings.Split(value, ","); len(dirs) > 1 {
log.Print("DEPRECATED: using comma-separated volume list.")
for _, dir := range dirs {
}
}
-// Delete deletes the block data from the unix storage
+// Trash trashes the block data from the unix storage
+// If trashLifetime == 0, the block is deleted
+// Else, the block is renamed as path/{loc}.trash.{deadline},
+// where deadline = now + trashLifetime
func (v *UnixVolume) Trash(loc string) error {
// Touch() must be called before calling Write() on a block. Touch()
// also uses lockfile(). This avoids a race condition between Write()
- // and Delete() because either (a) the file will be deleted and Touch()
+ // and Trash() because either (a) the file will be trashed and Touch()
// will signal to the caller that the file is not present (and needs to
// be re-written), or (b) Touch() will update the file's timestamp and
- // Delete() will read the correct up-to-date timestamp and choose not to
- // delete the file.
+ // Trash() will read the correct up-to-date timestamp and choose not to
+ // trash the file.
if v.readonly {
return MethodDisabledError
}
- if trashLifetime != 0 {
- return ErrNotImplemented
- }
if v.locker != nil {
v.locker.Lock()
defer v.locker.Unlock()
return nil
}
}
- return os.Remove(p)
+
+ if trashLifetime == 0 {
+ return os.Remove(p)
+ }
+ return os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(trashLifetime).Unix()))
}
// Untrash moves block from trash back into store
-// TBD
-func (v *UnixVolume) Untrash(loc string) error {
- return ErrNotImplemented
+// Look for path/{loc}.trash.{deadline} in storage,
+// and rename the first such file as path/{loc}
+func (v *UnixVolume) Untrash(loc string) (err error) {
+ if v.readonly {
+ return MethodDisabledError
+ }
+
+ files, err := ioutil.ReadDir(v.blockDir(loc))
+ if err != nil {
+ return err
+ }
+
+ if len(files) == 0 {
+ return os.ErrNotExist
+ }
+
+ foundTrash := false
+ prefix := fmt.Sprintf("%v.trash.", loc)
+ for _, f := range files {
+ if strings.HasPrefix(f.Name(), prefix) {
+ foundTrash = true
+ err = os.Rename(v.blockPath(f.Name()), v.blockPath(loc))
+ if err == nil {
+ break
+ }
+ }
+ }
+
+ if foundTrash == false {
+ return os.ErrNotExist
+ }
+
+ return
}
// blockDir returns the fully qualified directory name for the directory
return err
}
}
+
+var trashLocRegexp = regexp.MustCompile(`/([0-9a-f]{32})\.trash\.(\d+)$`)
+
+// EmptyTrash walks hierarchy looking for {hash}.trash.*
+// and deletes those with deadline < now.
+func (v *UnixVolume) EmptyTrash() {
+ var bytesDeleted, bytesInTrash int64
+ var blocksDeleted, blocksInTrash int
+
+ err := filepath.Walk(v.root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ log.Printf("EmptyTrash: filepath.Walk: %v: %v", path, err)
+ return nil
+ }
+ if info.Mode().IsDir() {
+ return nil
+ }
+ matches := trashLocRegexp.FindStringSubmatch(path)
+ if len(matches) != 3 {
+ return nil
+ }
+ deadline, err := strconv.ParseInt(matches[2], 10, 64)
+ if err != nil {
+ log.Printf("EmptyTrash: %v: ParseInt(%v): %v", path, matches[2], err)
+ return nil
+ }
+ bytesInTrash += info.Size()
+ blocksInTrash++
+ if deadline > time.Now().Unix() {
+ return nil
+ }
+ err = os.Remove(path)
+ if err != nil {
+ log.Printf("EmptyTrash: Remove %v: %v", path, err)
+ return nil
+ }
+ bytesDeleted += info.Size()
+ blocksDeleted++
+ return nil
+ })
+
+ if err != nil {
+ log.Printf("EmptyTrash error for %v: %v", v.String(), err)
+ }
+
+ log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+}
def _finished(self):
if self.subscribers is None:
raise Exception("Actor tried to finish twice")
- _notify_subscribers(self._later, self.subscribers)
+ _notify_subscribers(self.actor_ref.proxy(), self.subscribers)
self.subscribers = None
self._logger.info("finished")
def subscribe(self, subscriber):
if self.subscribers is None:
try:
- subscriber(self._later)
+ subscriber(self.actor_ref.proxy())
except pykka.ActorDeadError:
pass
else:
eligible = self.shutdown_eligible()
if eligible is True:
self._debug("Suggesting shutdown.")
- _notify_subscribers(self._later, self.subscribers)
+ _notify_subscribers(self.actor_ref.proxy(), self.subscribers)
elif self._shutdowns.window_open():
self._debug("Cannot shut down because %s", eligible)
elif self.last_shutdown_opening != next_opening:
first_ping_s = arvados_node.get('first_ping_at')
if (self.arvados_node is not None) or (not first_ping_s):
return None
- elif ((arvados_node['ip_address'] in self.cloud_node.private_ips) and
+ elif ((arvados_node['info'].get('ec2_instance_id') == self._cloud.node_id(self.cloud_node)) and
(arvados_timestamp(first_ping_s) >= self.cloud_node_start_time)):
self._later.update_arvados_node(arvados_node)
return self.cloud_node.id
self.SEARCH_CACHE[cache_key] = results[0]
return self.SEARCH_CACHE[cache_key]
- def list_nodes(self):
- return self.real.list_nodes(**self.list_kwargs)
+ def list_nodes(self, **kwargs):
+ l = self.list_kwargs.copy()
+ l.update(kwargs)
+ return self.real.list_nodes(**l)
def arvados_create_kwargs(self, size, arvados_node):
"""Return dynamic keyword arguments for create_node.
lambda self, value: setattr(self.real, attr_name, value),
doc=getattr(getattr(NodeDriver, attr_name), '__doc__', None))
+ # node id
+ @classmethod
+ def node_id(cls):
+ raise NotImplementedError("BaseComputeNodeDriver.node_id")
+
_locals = locals()
for _attr_name in dir(NodeDriver):
if (not _attr_name.startswith('_')) and (_attr_name not in _locals):
# Azure only supports filtering node lists by resource group.
# Do our own filtering based on tag.
nodes = [node for node in
- super(ComputeNodeDriver, self).list_nodes()
+ super(ComputeNodeDriver, self).list_nodes(ex_fetch_nic=False)
if node.extra["tags"].get("arvados-class") == self.tags["arvados-class"]]
for n in nodes:
# Need to populate Node.size
@classmethod
def node_start_time(cls, node):
return arvados_timestamp(node.extra["tags"].get("booted_at"))
+
+ @classmethod
+ def node_id(cls, node):
+ return node.name
time_str = node.extra['launch_time'].split('.', 2)[0] + 'UTC'
return time.mktime(time.strptime(
time_str,'%Y-%m-%dT%H:%M:%S%Z')) - time.timezone
+
+ @classmethod
+ def node_id(cls, node):
+ return node.id
node.extra['metadata']['items'], 'booted_at'))
except KeyError:
return 0
+
+ @classmethod
+ def node_id(cls, node):
+ return node.id
arvados_client=self._new_arvados(),
arvados_node=arvados_node,
cloud_client=self._new_cloud(),
- cloud_size=cloud_size).tell_proxy()
+ cloud_size=cloud_size).proxy()
self.booting[new_setup.actor_ref.actor_urn] = new_setup
self.sizes_booting_shutdown[new_setup.actor_ref.actor_urn] = cloud_size
shutdown = self._node_shutdown.start(
timer_actor=self._timer, cloud_client=self._new_cloud(),
arvados_client=self._new_arvados(),
- node_monitor=node_actor.actor_ref, cancellable=cancellable).proxy()
- self.shutdowns[cloud_node_id] = shutdown
+ node_monitor=node_actor.actor_ref, cancellable=cancellable)
+ self.shutdowns[cloud_node_id] = shutdown.proxy()
self.sizes_booting_shutdown[cloud_node_id] = cloud_node_obj.size
shutdown.tell_proxy().subscribe(self._later.node_finished_shutdown)
self.make_actor(2)
arv_node = testutil.arvados_node_mock(
2, hostname='compute-two.zzzzz.arvadosapi.com')
+ self.cloud_client.node_id.return_value = '2'
pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
self.assertEqual(self.cloud_mock.id, pair_id)
self.stop_proxy(self.node_actor)
self.driver_mock().create_node.side_effect = IOError
n = driver.create_node(testutil.MockSize(1), arv_node)
self.assertEqual('compute-000000000000001-zzzzz', n.name)
+
+ def test_ex_fetch_nic_false(self):
+ arv_node = testutil.arvados_node_mock(1, hostname=None)
+ driver = self.new_driver(create_kwargs={"tag_arvados-class": "dynamic-compute"})
+ nodelist = [testutil.cloud_node_mock(1, tags={"arvados-class": "dynamic-compute"})]
+ nodelist[0].name = 'compute-000000000000001-zzzzz'
+ self.driver_mock().list_nodes.return_value = nodelist
+ n = driver.list_nodes()
+ self.assertEqual(nodelist, n)
+ self.driver_mock().list_nodes.assert_called_with(ex_fetch_nic=False, ex_resource_group='TestResourceGroup')
self.cloud_factory().node_start_time.return_value = time.time()
self.cloud_updates = mock.MagicMock(name='updates_mock')
self.timer = testutil.MockTimer(deliver_immediately=False)
+ self.cloud_factory().node_id.side_effect = lambda node: node.id
self.node_setup = mock.MagicMock(name='setup_mock')
self.node_setup.start.side_effect = self.mock_node_start
def test_node_pairing_after_arvados_update(self):
cloud_node = testutil.cloud_node_mock(2)
self.make_daemon([cloud_node],
- [testutil.arvados_node_mock(2, ip_address=None)])
+ [testutil.arvados_node_mock(1, ip_address=None)])
arv_node = testutil.arvados_node_mock(2)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
'job_uuid': job_uuid,
'crunch_worker_state': crunch_worker_state,
'properties': {},
- 'info': {'ping_secret': 'defaulttestsecret'}}
+ 'info': {'ping_secret': 'defaulttestsecret', 'ec2_instance_id': str(node_num)}}
node.update(kwargs)
return node
projects / arvados.git / commitdiff