--- /dev/null
+export WORKSPACE?=$(shell pwd)
+help:
+ @echo >&2
+ @echo >&2 "There is no default make target here. Did you mean 'make test'?"
+ @echo >&2
+ @echo >&2 "More info:"
+ @echo >&2 " Installing --> http://doc.arvados.org/install"
+ @echo >&2 " Developing/contributing --> https://dev.arvados.org"
+ @echo >&2 " Project home --> https://arvados.org"
+ @echo >&2
+ @false
+test:
+ build/run-tests.sh ${TEST_FLAGS}
+packages:
+ build/run-build-packages-all-targets.sh ${PACKAGES_FLAGS}
+test-packages:
+ build/run-build-packages-all-targets.sh --test-packages ${PACKAGES_FLAGS}
+++ /dev/null
-Welcome to Arvados!
-
-The main Arvados web site is
- https://arvados.org
-
-The Arvados public wiki is located at
- https://dev.arvados.org/projects/arvados/wiki
-
-The Arvados public bug tracker is located at
- https://dev.arvados.org/projects/arvados/issues
-
-For support see
- http://doc.arvados.org/user/getting_started/community.html
-
-Installation documentation is located at
- http://doc.arvados.org/install
-
-If you wish to build the documentation yourself, follow the instructions in
-doc/README to build the documentation, then consult the "Install Guide".
-
-See COPYING for information about Arvados Free Software licenses.
--- /dev/null
+[Arvados](https://arvados.org) is a free software distributed computing platform
+for bioinformatics, data science, and high throughput analysis of massive data
+sets. Arvados supports a variety of cloud, cluster and HPC environments.
+
+Arvados consists of:
+
+* *Keep*: a petabyte-scale content-addressed distributed storage system for managing and
+ storing collections of files, accessible via HTTP and FUSE mount.
+
+* *Crunch*: a Docker-based cluster and HPC workflow engine designed providing
+ strong versioning, reproducibilty, and provenance of computations.
+
+* Related services and components including a web workbench for managing files
+ and compute jobs, REST APIs, SDKs, and other tools.
+
+## Quick start
+
+Curoverse maintains an Arvados public cloud demo at
+[https://cloud.curoverse.com](https://cloud.curoverse.com). A Google account
+is required to log in.
+
+To try out Arvados on your local workstation, you can use Arvbox, which
+provides Arvados components pre-installed in a Docker container (requires
+Docker 1.9+). After cloning the Arvados git repository:
+
+```
+$ cd arvados/tools/arvbox/bin
+$ ./arvbox start localdemo
+```
+
+In this mode you will only be able to connect to Arvbox from the same host. To
+configure Arvbox to be accessible over a network and for other options see
+http://doc.arvados.org/install/arvbox.html for details.
+
+## Documentation
+
+Complete documentation, including a User Guide, Installation documentation and
+API documentation is available at http://doc.arvados.org/
+
+If you wish to build the Arvados documentation from a local git clone, see
+doc/README.textile for instructions.
+
+## Community
+
+The [#arvados](irc://irc.oftc.net:6667/#arvados IRC) (Internet Relay Chat)
+channel at the
+[Open and Free Technology Community (irc.oftc.net)](http://www.oftc.net/oftc/)
+is available for live discussion and support. You can use a traditional IRC
+client or [join OFTC over the web.](https://webchat.oftc.net/?channels=arvados)
+
+The
+[Arvados user mailing list](http://lists.arvados.org/mailman/listinfo/arvados)
+is a forum for general discussion, questions, and news about Arvados
+development. The
+[Arvados developer mailing list](http://lists.arvados.org/mailman/listinfo/arvados-dev)
+is a forum for more technical discussion, intended for developers and
+contributers to Arvados.
+
+## Development
+
+[](https://ci.curoverse.com/job/arvados-api-server/)
+
+The Arvados public bug tracker is located at https://dev.arvados.org/projects/arvados/issues
+
+Continuous integration is hosted at https://ci.curoverse.com/
+
+Instructions for setting up a development environment and working on specific
+components can be found on the
+["Hacking Arvados" page of the Arvados wiki](https://dev.arvados.org/projects/arvados/wiki/Hacking).
+
+## Licensing
+
+Arvados is Free Software. See COPYING for information about Arvados Free
+Software licenses.
def textile_attributes
[ 'description' ]
end
+
+ def stderr_log_query(limit=nil)
+ query = Log.where(event_type: "stderr", object_uuid: self.uuid)
+ .order("id DESC")
+ query = query.limit(limit) if limit
+ query
+ end
+
+ def stderr_log_lines(limit=2000)
+ stderr_log_query(limit).results.reverse.
+ flat_map { |log| log.properties[:text].split("\n") rescue [] }
+ end
end
<div id="event_log_div"
class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
data-object-uuid="<%= @object.uuid %>"
- ></div>
+ ><%= @object.stderr_log_lines(Rails.configuration.running_job_log_records_to_fetch).join("\n") %>
+</div>
<%# Applying a long throttle suppresses the auto-refresh of this
partial that would normally be triggered by arv-log-event. %>
#
# The default setting (false) is appropriate for a multi-user site.
trust_all_content: false
+
+ # Maximum number of historic log records of a running job to fetch
+ # and display in the Log tab, while subscribing to web sockets.
+ running_job_log_records_to_fetch: 2000
datum = page.evaluate_script("jobGraphData[jobGraphData.length-1]['#{series}']")
assert_in_epsilon value, datum.to_f
end
+
+ test "test running job with just a few previous log records" do
+ Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+ job = Job.where(uuid: api_fixture("jobs")['running']['uuid']).results.first
+ visit page_with_token("admin", "/jobs/#{job.uuid}")
+
+ api = ArvadosApiClient.new
+
+ # Create just one old log record
+ api.api("logs", "", {log: {
+ object_uuid: job.uuid,
+ event_type: "stderr",
+ properties: {"text" => "Historic log message"}}})
+
+ click_link("Log")
+
+ # Expect "all" historic log records because we have less than
+ # default Rails.configuration.running_job_log_records_to_fetch count
+ assert_text 'Historic log message'
+
+ # Create new log record and expect it to show up in log tab
+ api.api("logs", "", {log: {
+ object_uuid: job.uuid,
+ event_type: "stderr",
+ properties: {"text" => "Log message after subscription"}}})
+ assert_text 'Log message after subscription'
+ end
+
+ test "test running job with too many previous log records" do
+ Rails.configuration.running_job_log_records_to_fetch = 5
+
+ Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+ job = Job.where(uuid: api_fixture("jobs")['running']['uuid']).results.first
+
+ visit page_with_token("admin", "/jobs/#{job.uuid}")
+
+ api = ArvadosApiClient.new
+
+ # Create Rails.configuration.running_job_log_records_to_fetch + 1 log records
+ (0..Rails.configuration.running_job_log_records_to_fetch).each do |count|
+ api.api("logs", "", {log: {
+ object_uuid: job.uuid,
+ event_type: "stderr",
+ properties: {"text" => "Old log message #{count}"}}})
+ end
+
+ # Go to log tab, which results in subscribing to websockets
+ click_link("Log")
+
+ # Expect all but the first historic log records,
+ # because that was one too many than fetch count.
+ (1..Rails.configuration.running_job_log_records_to_fetch).each do |count|
+ assert_text "Old log message #{count}"
+ end
+ assert_no_text 'Old log message 0'
+
+ # Create one more log record after subscription
+ api.api("logs", "", {log: {
+ object_uuid: job.uuid,
+ event_type: "stderr",
+ properties: {"text" => "Life goes on!"}}})
+ # Expect it to show up in log tab
+ assert_text 'Life goes on!'
+ end
end
# FIXME: Remove this line after #6885 is done.
fpm_args+=(--iteration 2)
-
-# FIXME: Remove once support for llfuse 0.42+ is in place
-fpm_args+=(-v 0.41.1)
--- /dev/null
+#!/bin/bash
+
+build=$1
+file=$2
+outputdir=$3
+
+usage() {
+ echo "./$0 build_number file_to_parse output_dir"
+ echo "this script will use the build output to generate *csv and *txt"
+ echo "for jenkins plugin plot https://github.com/jenkinsci/plot-plugin/"
+}
+
+if [ $# -ne 3 ]
+then
+ usage
+ exit 1
+fi
+
+if [ ! -e $file ]
+then
+ usage
+ echo "$file doesn't exist! exiting"
+ exit 2
+fi
+if [ ! -w $outputdir ]
+then
+ usage
+ echo "$outputdir isn't writeable! exiting"
+ exit 3
+fi
+
+#------------------------------
+## MAXLINE is the amount of lines that will read after the pattern
+## is match (the logfile could be hundred thousands lines long).
+## 1000 should be safe enough to capture all the output of the individual test
+MAXLINES=1000
+
+## TODO: check $build and $file make sense
+
+for test in \
+ test_Create_and_show_large_collection_with_manifest_text_of_20000000 \
+ test_Create,_show,_and_update_description_for_large_collection_with_manifest_text_of_100000 \
+ test_Create_one_large_collection_of_20000000_and_one_small_collection_of_10000_and_combine_them
+do
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ (zgrep -i -E -A$MAXLINES "^[A-Za-z0-9]+Test: $test" $file && echo "----") | tail -n +1 | tail --lines=+3|grep -B$MAXLINES -E "^-*$" -m1 > $outputdir/$cleaned_test-$build.txt
+ result=$?
+ if [ $result -eq 0 ]
+ then
+ echo processing $outputdir/$cleaned_test-$build.txt creating $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed (.*) in [0-9]+ms.*$/;print "".++$line."-$1,";' | perl -p -e 's/,$//g'|tr " " "_" ) > $outputdir/$cleaned_test.csv
+ echo $(grep ^Completed $outputdir/$cleaned_test-$build.txt | perl -n -e '/^Completed.*in ([0-9]+)ms.*$/;print "$1,";' | perl -p -e 's/,$//g' ) >> $outputdir/$cleaned_test.csv
+ #echo URL=https://ci.curoverse.com/view/job/arvados-api-server/ws/apps/workbench/log/$cleaned_test-$build.txt/*view*/ >> $outputdir/$test.properties
+ else
+ echo "$test was't found on $file"
+ cleaned_test=$(echo $test | tr -d ",.:;/")
+ > $outputdir/$cleaned_test.csv
+ fi
+done
--- /dev/null
+LIBCLOUD_PIN=0.20.2.dev1
\ No newline at end of file
--- /dev/null
+*/generated
+common-generated/
--- /dev/null
+all: centos6/generated debian7/generated debian8/generated ubuntu1204/generated ubuntu1404/generated
+
+centos6/generated: common-generated-all
+ test -d centos6/generated || mkdir centos6/generated
+ cp -rlt centos6/generated common-generated/*
+
+debian7/generated: common-generated-all
+ test -d debian7/generated || mkdir debian7/generated
+ cp -rlt debian7/generated common-generated/*
+
+debian8/generated: common-generated-all
+ test -d debian8/generated || mkdir debian8/generated
+ cp -rlt debian8/generated common-generated/*
+
+ubuntu1204/generated: common-generated-all
+ test -d ubuntu1204/generated || mkdir ubuntu1204/generated
+ cp -rlt ubuntu1204/generated common-generated/*
+
+ubuntu1404/generated: common-generated-all
+ test -d ubuntu1404/generated || mkdir ubuntu1404/generated
+ cp -rlt ubuntu1404/generated common-generated/*
+
+common-generated-all: common-generated/golang-amd64.tar.gz
+
+common-generated/golang-amd64.tar.gz: common-generated
+ wget -cqO common-generated/golang-amd64.tar.gz http://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
+
+common-generated:
+ mkdir common-generated
--- /dev/null
+==================
+DOCKER IMAGE BUILD
+==================
+
+1. `make`
+2. `cd DISTRO`
+3. `docker build -t arvados/build:DISTRO .`
+
+==============
+BUILD PACKAGES
+==============
+
+`docker run -v /path/to/your/arvados-dev/jenkins:/jenkins -v /path/to/your/arvados:/arvados arvados/build:DISTRO`
--- /dev/null
+#!/bin/bash
+
+make
+
+for target in `find -maxdepth 1 -type d |grep -v generated`; do
+ if [[ "$target" == "." ]]; then
+ continue
+ fi
+ target=${target#./}
+ echo $target
+ cd $target
+ docker build -t arvados/build:$target .
+ cd ..
+done
+
+
--- /dev/null
+FROM centos:6
+MAINTAINER Brett Smith <brett@curoverse.com>
+
+# Install build dependencies provided in base distribution
+RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar scl-utils centos-release-SCL postgresql-devel
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Need to "touch" RPM database to workaround bug in interaction between
+# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
+RUN touch /var/lib/rpm/* && yum -q -y install python27 python33
+RUN scl enable python33 "easy_install-3.3 pip" && scl enable python27 "easy_install-2.7 pip"
+
+RUN cd /tmp && \
+ curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
+ rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
+ sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
+
+RUN touch /var/lib/rpm/* && yum install --assumeyes git
+
+ENV WORKSPACE /arvados
+CMD ["scl", "enable", "python33", "python27", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos6"]
--- /dev/null
+FROM debian:wheezy
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian7"]
--- /dev/null
+FROM debian:jessie
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
--- /dev/null
+FROM ubuntu:precise
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1204"]
--- /dev/null
+FROM ubuntu:trusty
+MAINTAINER Brett Smith <brett@curoverse.com>
+
+# Install dependencies and set up system.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip
+
+# Install RVM
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundler fpm
+
+# Install golang binary
+ADD generated/golang-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
--- /dev/null
+FROM centos:6
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+RUN yum -q install --assumeyes scl-utils centos-release-SCL \
+ which tar
+
+# Install RVM
+RUN touch /var/lib/rpm/* && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1 && \
+ /usr/local/rvm/bin/rvm-exec default gem install bundle fpm
+
+RUN cd /tmp && \
+ curl -OL 'http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm' && \
+ rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm && \
+ sed -i 's/enabled = 0/enabled = 1/' /etc/yum.repos.d/rpmforge.repo
+
+COPY localrepo.repo /etc/yum.repos.d/localrepo.repo
\ No newline at end of file
--- /dev/null
+[localrepo]
+name=Arvados Test
+baseurl=file:///arvados/packages/centos6
+gpgcheck=0
+enabled=1
--- /dev/null
+FROM debian:7
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl procps && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian7/ /" >>/etc/apt/sources.list
--- /dev/null
+FROM debian:8
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
--- /dev/null
+FROM ubuntu:precise
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/ubuntu1204/ /" >>/etc/apt/sources.list
\ No newline at end of file
--- /dev/null
+FROM ubuntu:trusty
+MAINTAINER Peter Amstutz <peter.amstutz@curoverse.com>
+
+# Install RVM
+RUN apt-get update && apt-get -y install curl && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.1
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+
+set -eu
+
+FAIL=0
+
+echo
+
+while read so && [ -n "$so" ]; do
+ if ldd "$so" | grep "not found" ; then
+ echo "^^^ Missing while scanning $so ^^^"
+ FAIL=1
+ fi
+done <<EOF
+$(find -name '*.so')
+EOF
+
+if test -x "/jenkins/package-testing/test-package-$1.sh" ; then
+ if ! "/jenkins/package-testing/test-package-$1.sh" ; then
+ FAIL=1
+ fi
+fi
+
+if test $FAIL = 0 ; then
+ echo "Package $1 passed"
+fi
+
+exit $FAIL
--- /dev/null
+#!/bin/bash
+
+set -eu
+
+# Multiple .deb based distros symlink to this script, so extract the target
+# from the invocation path.
+target=$(echo $0 | sed 's/.*test-packages-\([^.]*\)\.sh.*/\1/')
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+dpkg-query --show > "$ARV_PACKAGES_DIR/$1.before"
+
+apt-get -qq update
+apt-get --assume-yes --force-yes install "$1"
+
+dpkg-query --show > "$ARV_PACKAGES_DIR/$1.after"
+
+set +e
+diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" > "$ARV_PACKAGES_DIR/$1.diff"
+set -e
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+export ARV_PACKAGES_DIR="/arvados/packages/$target"
+
+dpkg-deb -x $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) .
+
+while read so && [ -n "$so" ]; do
+ echo
+ echo "== Packages dependencies for $so =="
+ ldd "$so" | awk '($3 ~ /^\//){print $3}' | sort -u | xargs dpkg -S | cut -d: -f1 | sort -u
+done <<EOF
+$(find -name '*.so')
+EOF
+
+exec /jenkins/package-testing/common-test-packages.sh "$1"
--- /dev/null
+#!/bin/sh
+set -e
+cd /var/www/arvados-api/current/
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ apt-get install -y nginx
+ dpkg-reconfigure arvados-api-server
+ ;;
+ centos6)
+ yum install --assumeyes httpd
+ yum reinstall --assumeyes arvados-api-server
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-api-server.gems"
--- /dev/null
+#!/bin/sh
+exec python <<EOF
+import libcloud.compute.types
+import libcloud.compute.providers
+libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.AZURE_ARM)
+print "Successfully imported compatible libcloud library"
+EOF
--- /dev/null
+#!/bin/bash
+
+set -e
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ FORMAT=deb
+ ;;
+ centos6)
+ FORMAT=rpm
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+title () {
+ txt="********** $1 **********"
+ printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
+}
+
+checkexit() {
+ if [[ "$1" != "0" ]]; then
+ title "!!!!!! $2 FAILED !!!!!!"
+ fi
+}
+
+
+# Find the SSO server package
+
+cd "$WORKSPACE"
+
+if [[ ! -d "/var/www/arvados-sso" ]]; then
+ echo "/var/www/arvados-sso should exist"
+ exit 1
+fi
+
+if [[ ! -e "/etc/arvados/sso/application.yml" ]]; then
+ mkdir -p /etc/arvados/sso/
+ RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
+ cp config/application.yml.example /etc/arvados/sso/application.yml
+ sed -i -e 's/uuid_prefix: ~/uuid_prefix: zzzzz/' /etc/arvados/sso/application.yml
+ sed -i -e "s/secret_token: ~/secret_token: $RANDOM_PASSWORD/" /etc/arvados/sso/application.yml
+fi
+
+if [[ ! -e "/etc/arvados/sso/database.yml" ]]; then
+ # We need to set up our database configuration now.
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # postgres packaging on CentOS6 is kind of primitive, needs an initdb
+ service postgresql initdb
+ if [ "$TARGET" = "centos6" ]; then
+ sed -i -e "s/127.0.0.1\/32 ident/127.0.0.1\/32 md5/" /var/lib/pgsql/data/pg_hba.conf
+ sed -i -e "s/::1\/128 ident/::1\/128 md5/" /var/lib/pgsql/data/pg_hba.conf
+ fi
+ fi
+ service postgresql start
+
+ RANDOM_PASSWORD=`date | md5sum |cut -f1 -d' '`
+ cat >/etc/arvados/sso/database.yml <<EOF
+production:
+ adapter: postgresql
+ encoding: utf8
+ database: sso_provider_production
+ username: sso_provider_user
+ password: $RANDOM_PASSWORD
+ host: localhost
+EOF
+
+ su postgres -c "psql -c \"CREATE USER sso_provider_user WITH PASSWORD '$RANDOM_PASSWORD'\""
+ su postgres -c "createdb sso_provider_production -O sso_provider_user"
+fi
+
+if [[ "$FORMAT" == "deb" ]]; then
+ # Test 2: the package should reconfigure cleanly
+ dpkg-reconfigure arvados-sso-server || EXITCODE=3
+
+ cd /var/www/arvados-sso/current/
+ /usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-sso-server.gems"
+
+ # Test 3: the package should remove cleanly
+ apt-get remove arvados-sso-server --yes || EXITCODE=3
+
+ checkexit $EXITCODE "apt-get remove arvados-sso-server --yes"
+
+ # Test 4: the package configuration should remove cleanly
+ dpkg --purge arvados-sso-server || EXITCODE=4
+
+ checkexit $EXITCODE "dpkg --purge arvados-sso-server"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=4
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+ # Test 5: the package should remove cleanly with --purge
+
+ apt-get remove arvados-sso-server --purge --yes || EXITCODE=5
+
+ checkexit $EXITCODE "apt-get remove arvados-sso-server --purge --yes"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=5
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+elif [[ "$FORMAT" == "rpm" ]]; then
+
+ # Set up Nginx first
+ # (courtesy of https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/el6/install_passenger.html)
+ yum install -q -y epel-release pygpgme curl
+ curl --fail -sSLo /etc/yum.repos.d/passenger.repo https://oss-binaries.phusionpassenger.com/yum/definitions/el-passenger.repo
+ yum install -q -y nginx passenger
+ sed -i -e 's/^# passenger/passenger/' /etc/nginx/conf.d/passenger.conf
+ # Done setting up Nginx
+
+ # Test 2: the package should reinstall cleanly
+ yum --assumeyes reinstall arvados-sso-server || EXITCODE=3
+
+ cd /var/www/arvados-sso/current/
+ /usr/local/rvm/bin/rvm-exec default bundle list >$ARV_PACKAGES_DIR/arvados-sso-server.gems
+
+ # Test 3: the package should remove cleanly
+ yum -q -y remove arvados-sso-server || EXITCODE=3
+
+ checkexit $EXITCODE "yum -q -y remove arvados-sso-server"
+
+ if [[ -e "/var/www/arvados-sso" ]]; then
+ EXITCODE=3
+ fi
+
+ checkexit $EXITCODE "leftover items under /var/www/arvados-sso"
+
+fi
+
+if [[ "$EXITCODE" == "0" ]]; then
+ echo "Testing complete, no errors!"
+else
+ echo "Errors while testing!"
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/sh
+set -e
+cd /var/www/arvados-workbench/current/
+
+case "$TARGET" in
+ debian*|ubuntu*)
+ apt-get install -y nginx
+ dpkg-reconfigure arvados-workbench
+ ;;
+ centos6)
+ yum install --assumeyes httpd
+ yum reinstall --assumeyes arvados-workbench
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+/usr/local/rvm/bin/rvm-exec default bundle list >"$ARV_PACKAGES_DIR/arvados-workbench.gems"
--- /dev/null
+#!/bin/sh
+
+exec python <<EOF
+import arvados_fuse
+print "Successfully imported arvados_fuse"
+EOF
--- /dev/null
+#!/bin/sh
+
+exec python <<EOF
+import arvados
+print "Successfully imported arvados"
+EOF
--- /dev/null
+#!/bin/bash
+
+set -eu
+
+yum -q clean all
+touch /var/lib/rpm/*
+
+export ARV_PACKAGES_DIR=/arvados/packages/centos6
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.before"
+
+yum install --assumeyes $1
+
+rpm -qa | sort > "$ARV_PACKAGES_DIR/$1.after"
+
+set +e
+diff "$ARV_PACKAGES_DIR/$1.before" "$ARV_PACKAGES_DIR/$1.after" >"$ARV_PACKAGES_DIR/$1.diff"
+set -e
+
+SCL=""
+if scl enable python27 true 2>/dev/null ; then
+ SCL="scl enable python27"
+fi
+
+mkdir -p /tmp/opts
+cd /tmp/opts
+
+rpm2cpio $(ls -t "$ARV_PACKAGES_DIR/$1"-*.rpm | head -n1) | cpio -idm 2>/dev/null
+
+shared=$(find -name '*.so')
+if test -n "$shared" ; then
+ for so in $shared ; do
+ echo
+ echo "== Packages dependencies for $so =="
+ $SCL ldd "$so" \
+ | awk '($3 ~ /^\//){print $3}' | sort -u | xargs rpm -qf | sort -u
+ done
+fi
+
+if test -n "$SCL" ; then
+ exec $SCL "/jenkins/package-testing/common-test-packages.sh '$1'"
+else
+ exec /jenkins/package-testing/common-test-packages.sh "$1"
+fi
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
--- /dev/null
+When run-build-packages.sh builds a Rails package, it generates the package's pre/post-inst/rm scripts by concatenating:
+
+1. package_name.sh, which defines variables about where package files live and some human-readable names about them.
+2. step2.sh, which uses those to define some utility variables and set defaults for things that aren't set.
+3. stepname.sh, like postinst.sh, prerm.sh, etc., which uses all this information to do the actual work.
+
+Since our build process is a tower of shell scripts, concatenating files seemed like the least worst option to share code between these files and packages. More advanced code generation would've been too much trouble to integrate into our build process at this time. Trying to inject portions of files into other files seemed error-prone and likely to introduce bugs to the end result.
+
+postinst.sh lets the early parts define a few hooks to control behavior:
+
+* After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles. By default this is a noop function (in step2.sh). API server defines this to set up the old omniauth.rb conffile.
+* Before it restarts nginx, it calls setup_before_nginx_restart. By default this is a noop function (in step2.sh). API server defines this to set up the internal git repository, if necessary.
+* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. SSO server uses db:schema:load. Workbench doesn't set this, which causes the postinst to skip all database work.
+* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task. SSO clears this flag (it doesn't have that task code).
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-api-server
+INSTALL_PATH=/var/www/arvados-api
+CONFIG_PATH=/etc/arvados/api
+DOC_URL="http://doc.arvados.org/install/install-api-server.html#configure"
+
+RAILSPKG_DATABASE_LOAD_TASK=db:structure:load
+setup_extra_conffiles() {
+ setup_conffile initializers/omniauth.rb
+}
+
+setup_before_nginx_restart() {
+ # initialize git_internal_dir
+ # usually /var/lib/arvados/internal.git (set in application.default.yml )
+ if [ "$APPLICATION_READY" = "1" ]; then
+ GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:check 2>&1 | grep git_internal_dir | awk '{ print $2 }')
+ if [ ! -e "$GIT_INTERNAL_DIR" ]; then
+ run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
+ mkdir -p "$GIT_INTERNAL_DIR"
+ run_and_report "Initializing git_internal_dir '$GIT_INTERNAL_DIR'" \
+ git init --quiet --bare $GIT_INTERNAL_DIR
+ else
+ echo "Initializing git_internal_dir $GIT_INTERNAL_DIR: directory exists, skipped."
+ fi
+ run_and_report "Making sure '$GIT_INTERNAL_DIR' has the right permission" \
+ chown -R "$WWW_OWNER:" "$GIT_INTERNAL_DIR"
+ else
+ echo "Initializing git_internal_dir... skipped."
+ fi
+}
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-sso-server
+INSTALL_PATH=/var/www/arvados-sso
+CONFIG_PATH=/etc/arvados/sso
+DOC_URL="http://doc.arvados.org/install/install-sso.html#configure"
+RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
+RAILSPKG_SUPPORTS_CONFIG_CHECK=0
--- /dev/null
+#!/bin/sh
+# This file declares variables common to all scripts for one Rails package.
+
+PACKAGE_NAME=arvados-workbench
+INSTALL_PATH=/var/www/arvados-workbench
+CONFIG_PATH=/etc/arvados/workbench
+DOC_URL="http://doc.arvados.org/install/install-workbench-app.html#configure"
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+set -e
+
+DATABASE_READY=1
+APPLICATION_READY=1
+
+if [ -s "$HOME/.rvm/scripts/rvm" ] || [ -s "/usr/local/rvm/scripts/rvm" ]; then
+ COMMAND_PREFIX="/usr/local/rvm/bin/rvm-exec default"
+else
+ COMMAND_PREFIX=
+fi
+
+report_not_ready() {
+ local ready_flag="$1"; shift
+ local config_file="$1"; shift
+ if [ "1" != "$ready_flag" ]; then cat >&2 <<EOF
+
+PLEASE NOTE:
+
+The $PACKAGE_NAME package was not configured completely because
+$config_file needs some tweaking.
+Please refer to the documentation at
+<$DOC_URL> for more details.
+
+When $(basename "$config_file") has been modified,
+reconfigure or reinstall this package.
+
+EOF
+ fi
+}
+
+report_web_service_warning() {
+ local warning="$1"; shift
+ cat >&2 <<EOF
+
+WARNING: $warning.
+
+To override, set the WEB_SERVICE environment variable to the name of the service
+hosting the Rails server.
+
+For Debian-based systems, then reconfigure this package with dpkg-reconfigure.
+
+For RPM-based systems, then reinstall this package.
+
+EOF
+}
+
+run_and_report() {
+ # Usage: run_and_report ACTION_MSG CMD
+ # This is the usual wrapper that prints ACTION_MSG, runs CMD, then writes
+ # a message about whether CMD succeeded or failed. Returns the exit code
+ # of CMD.
+ local action_message="$1"; shift
+ local retcode=0
+ echo -n "$action_message..."
+ if "$@"; then
+ echo " done."
+ else
+ retcode=$?
+ echo " failed."
+ fi
+ return $retcode
+}
+
+setup_confdirs() {
+ for confdir in "$@"; do
+ if [ ! -d "$confdir" ]; then
+ install -d -g "$WWW_OWNER" -m 0750 "$confdir"
+ fi
+ done
+}
+
+setup_conffile() {
+ # Usage: setup_conffile CONFFILE_PATH [SOURCE_PATH]
+ # Both paths are relative to RELEASE_CONFIG_PATH.
+ # This function will try to safely ensure that a symbolic link for
+ # the configuration file points from RELEASE_CONFIG_PATH to CONFIG_PATH.
+ # If SOURCE_PATH is given, this function will try to install that file as
+ # the configuration file in CONFIG_PATH, and return 1 if the file in
+ # CONFIG_PATH is unmodified from the source.
+ local conffile_relpath="$1"; shift
+ local conffile_source="$1"
+ local release_conffile="$RELEASE_CONFIG_PATH/$conffile_relpath"
+ local etc_conffile="$CONFIG_PATH/$(basename "$conffile_relpath")"
+
+ # Note that -h can return true and -e will return false simultaneously
+ # when the target is a dangling symlink. We're okay with that outcome,
+ # so check -h first.
+ if [ ! -h "$release_conffile" ]; then
+ if [ ! -e "$release_conffile" ]; then
+ ln -s "$etc_conffile" "$release_conffile"
+ # If there's a config file in /var/www identical to the one in /etc,
+ # overwrite it with a symlink after porting its permissions.
+ elif cmp --quiet "$release_conffile" "$etc_conffile"; then
+ local ownership="$(stat -c "%u:%g" "$release_conffile")"
+ local owning_group="${ownership#*:}"
+ if [ 0 != "$owning_group" ]; then
+ chgrp "$owning_group" "$CONFIG_PATH" /etc/arvados
+ fi
+ chown "$ownership" "$etc_conffile"
+ chmod --reference="$release_conffile" "$etc_conffile"
+ ln --force -s "$etc_conffile" "$release_conffile"
+ fi
+ fi
+
+ if [ -n "$conffile_source" ]; then
+ if [ ! -e "$etc_conffile" ]; then
+ install -g "$WWW_OWNER" -m 0640 \
+ "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"
+ return 1
+ # Even if $etc_conffile already existed, it might be unmodified from
+ # the source. This is especially likely when a user installs, updates
+ # database.yml, then reconfigures before they update application.yml.
+ # Use cmp to be sure whether $etc_conffile is modified.
+ elif cmp --quiet "$RELEASE_CONFIG_PATH/$conffile_source" "$etc_conffile"; then
+ return 1
+ fi
+ fi
+}
+
+prepare_database() {
+ DB_MIGRATE_STATUS=`$COMMAND_PREFIX bundle exec rake db:migrate:status 2>&1 || true`
+ if echo $DB_MIGRATE_STATUS | grep -qF 'Schema migrations table does not exist yet.'; then
+ # The database exists, but the migrations table doesn't.
+ run_and_report "Setting up database" $COMMAND_PREFIX bundle exec \
+ rake "$RAILSPKG_DATABASE_LOAD_TASK" db:seed
+ elif echo $DB_MIGRATE_STATUS | grep -q '^database: '; then
+ run_and_report "Running db:migrate" \
+ $COMMAND_PREFIX bundle exec rake db:migrate
+ elif echo $DB_MIGRATE_STATUS | grep -q 'database .* does not exist'; then
+ if ! run_and_report "Running db:setup" \
+ $COMMAND_PREFIX bundle exec rake db:setup 2>/dev/null; then
+ echo "Warning: unable to set up database." >&2
+ DATABASE_READY=0
+ fi
+ else
+ echo "Warning: Database is not ready to set up. Skipping database setup." >&2
+ DATABASE_READY=0
+ fi
+}
+
+configure_version() {
+ WEB_SERVICE=${WEB_SERVICE:-$(service --status-all 2>/dev/null \
+ | grep -Eo '\bnginx|httpd[^[:space:]]*' || true)}
+ if [ -z "$WEB_SERVICE" ]; then
+ report_web_service_warning "Web service (Nginx or Apache) not found"
+ elif [ "$WEB_SERVICE" != "$(echo "$WEB_SERVICE" | head -n 1)" ]; then
+ WEB_SERVICE=$(echo "$WEB_SERVICE" | head -n 1)
+ report_web_service_warning \
+ "Multiple web services found. Choosing the first one ($WEB_SERVICE)"
+ fi
+
+ if [ -e /etc/redhat-release ]; then
+ # Recognize any service that starts with "nginx"; e.g., nginx16.
+ if [ "$WEB_SERVICE" != "${WEB_SERVICE#nginx}" ]; then
+ WWW_OWNER=nginx
+ else
+ WWW_OWNER=apache
+ fi
+ else
+ # Assume we're on a Debian-based system for now.
+ # Both Apache and Nginx run as www-data by default.
+ WWW_OWNER=www-data
+ fi
+
+ echo
+ echo "Assumption: $WEB_SERVICE is configured to serve Rails from"
+ echo " $RELEASE_PATH"
+ echo "Assumption: $WEB_SERVICE and passenger run as $WWW_OWNER"
+ echo
+
+ echo -n "Creating symlinks to configuration in $CONFIG_PATH ..."
+ setup_confdirs /etc/arvados "$CONFIG_PATH"
+ setup_conffile environments/production.rb environments/production.rb.example \
+ || true
+ setup_conffile application.yml application.yml.example || APPLICATION_READY=0
+ if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
+ setup_conffile database.yml database.yml.example || DATABASE_READY=0
+ fi
+ setup_extra_conffiles
+ echo "... done."
+
+ # Before we do anything else, make sure some directories and files are in place
+ if [ ! -e $SHARED_PATH/log ]; then mkdir -p $SHARED_PATH/log; fi
+ if [ ! -e $RELEASE_PATH/tmp ]; then mkdir -p $RELEASE_PATH/tmp; fi
+ if [ ! -e $RELEASE_PATH/log ]; then ln -s $SHARED_PATH/log $RELEASE_PATH/log; fi
+ if [ ! -e $SHARED_PATH/log/production.log ]; then touch $SHARED_PATH/log/production.log; fi
+
+ cd "$RELEASE_PATH"
+ export RAILS_ENV=production
+
+ if ! $COMMAND_PREFIX bundle --version >/dev/null; then
+ run_and_report "Installing bundle" $COMMAND_PREFIX gem install bundle
+ fi
+
+ run_and_report "Running bundle install" \
+ $COMMAND_PREFIX bundle install --path $SHARED_PATH/vendor_bundle --local --quiet
+
+ echo -n "Ensuring directory and file permissions ..."
+ # Ensure correct ownership of a few files
+ chown "$WWW_OWNER:" $RELEASE_PATH/config/environment.rb
+ chown "$WWW_OWNER:" $RELEASE_PATH/config.ru
+ chown "$WWW_OWNER:" $RELEASE_PATH/Gemfile.lock
+ chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
+ chown -R "$WWW_OWNER:" $SHARED_PATH/log
+ case "$RAILSPKG_DATABASE_LOAD_TASK" in
+ db:schema:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/schema.rb ;;
+ db:structure:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/structure.sql ;;
+ esac
+ chmod 644 $SHARED_PATH/log/*
+ chmod -R 2775 $RELEASE_PATH/tmp
+ echo "... done."
+
+ if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
+ prepare_database
+ fi
+
+ if [ 11 = "$RAILSPKG_SUPPORTS_CONFIG_CHECK$APPLICATION_READY" ]; then
+ run_and_report "Checking application.yml for completeness" \
+ $COMMAND_PREFIX bundle exec rake config:check || APPLICATION_READY=0
+ fi
+
+ # precompile assets; thankfully this does not take long
+ if [ "$APPLICATION_READY" = "1" ]; then
+ run_and_report "Precompiling assets" \
+ $COMMAND_PREFIX bundle exec rake assets:precompile -q -s 2>/dev/null \
+ || APPLICATION_READY=0
+ else
+ echo "Precompiling assets... skipped."
+ fi
+ chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
+
+ setup_before_nginx_restart
+
+ if [ ! -z "$WEB_SERVICE" ]; then
+ service "$WEB_SERVICE" restart
+ fi
+}
+
+if [ "$1" = configure ]; then
+ # This is a debian-based system
+ configure_version
+elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
+ # This is an rpm-based system
+ configure_version
+fi
+
+report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
+report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+set -e
+
+purge () {
+ rm -rf $SHARED_PATH/vendor_bundle
+ rm -rf $SHARED_PATH/log
+ rm -rf $CONFIG_PATH
+ rmdir $SHARED_PATH || true
+ rmdir $INSTALL_PATH || true
+}
+
+if [ "$1" = 'purge' ]; then
+ # This is a debian-based system and purge was requested
+ purge
+elif [ "$1" = "0" ]; then
+ # This is an rpm-based system, no guarantees are made, always purge
+ # Apparently yum doesn't actually remember what it installed.
+ # Clean those files up here, then purge.
+ rm -rf $RELEASE_PATH
+ purge
+fi
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions and step2.sh.
+
+remove () {
+ rm -f $RELEASE_PATH/config/database.yml
+ rm -f $RELEASE_PATH/config/environments/production.rb
+ rm -f $RELEASE_PATH/config/application.yml
+ # Old API server configuration file.
+ rm -f $RELEASE_PATH/config/initializers/omniauth.rb
+ rm -rf $RELEASE_PATH/public/assets/
+ rm -rf $RELEASE_PATH/tmp
+ rm -rf $RELEASE_PATH/.bundle
+ rm -rf $RELEASE_PATH/log
+}
+
+if [ "$1" = 'remove' ]; then
+ # This is a debian-based system and removal was requested
+ remove
+elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
+ # This is an rpm-based system
+ remove
+fi
--- /dev/null
+#!/bin/sh
+# This code runs after package variable definitions, before the actual
+# pre/post package work, to set some variable and function defaults.
+
+if [ -z "$INSTALL_PATH" ]; then
+ cat >&2 <<EOF
+
+PACKAGE BUILD ERROR: $0 is missing package metadata.
+
+This package is buggy. Please mail <support@curoverse.com> to let
+us know the name and version number of the package you tried to
+install, and we'll get it fixed.
+
+EOF
+ exit 3
+fi
+
+RELEASE_PATH=$INSTALL_PATH/current
+RELEASE_CONFIG_PATH=$RELEASE_PATH/config
+SHARED_PATH=$INSTALL_PATH/shared
+
+RAILSPKG_SUPPORTS_CONFIG_CHECK=${RAILSPKG_SUPPORTS_CONFIG_CHECK:-1}
+if ! type setup_extra_conffiles >/dev/null 2>&1; then
+ setup_extra_conffiles() { return; }
+fi
+if ! type setup_before_nginx_restart >/dev/null 2>&1; then
+ setup_before_nginx_restart() { return; }
+fi
--- /dev/null
+#!/bin/bash
+
+function usage {
+ echo >&2
+ echo >&2 "usage: $0 [options]"
+ echo >&2
+ echo >&2 "$0 options:"
+ echo >&2 " -t, --tags [csv_tags] comma separated tags"
+ echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2
+ echo >&2 " If no options are given, just builds the images."
+}
+
+upload=false
+
+# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+TEMP=`getopt -o hut: \
+ --long help,upload,tags: \
+ -n "$0" -- "$@"`
+
+if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while [ $# -ge 1 ]
+do
+ case $1 in
+ -u | --upload)
+ upload=true
+ shift
+ ;;
+ -t | --tags)
+ case "$2" in
+ "")
+ echo "ERROR: --tags needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ tags=$2;
+ shift 2
+ ;;
+ esac
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+
+EXITCODE=0
+
+COLUMNS=80
+
+title () {
+ printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
+}
+
+docker_push () {
+ if [[ ! -z "$tags" ]]
+ then
+ for tag in $( echo $tags|tr "," " " )
+ do
+ $DOCKER tag $1 $1:$tag
+ done
+ fi
+
+ # Sometimes docker push fails; retry it a few times if necessary.
+ for i in `seq 1 5`; do
+ $DOCKER push $*
+ ECODE=$?
+ if [[ "$ECODE" == "0" ]]; then
+ break
+ fi
+ done
+
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker push $* failed !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+echo $WORKSPACE
+
+# find the docker binary
+DOCKER=`which docker.io`
+
+if [[ "$DOCKER" == "" ]]; then
+ DOCKER=`which docker`
+fi
+
+if [[ "$DOCKER" == "" ]]; then
+ title "Error: you need to have docker installed. Could not find the docker executable."
+ exit 1
+fi
+
+# DOCKER
+title "Starting docker build"
+
+timer_reset
+
+# clean up the docker build environment
+cd "$WORKSPACE"
+
+tools/arvbox/bin/arvbox build dev
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+tools/arvbox/bin/arvbox build localdemo
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+title "docker build complete (`timer`)"
+
+title "uploading images"
+
+timer_reset
+
+if [[ "$ECODE" != "0" ]]; then
+ title "upload arvados images SKIPPED because build failed"
+else
+ if [[ $upload == true ]]; then
+ ## 20150526 nico -- *sometimes* dockerhub needs re-login
+ ## even though credentials are already in .dockercfg
+ docker login -u arvados
+
+ docker_push arvados/arvbox-dev
+ docker_push arvados/arvbox-demo
+ title "upload arvados images complete (`timer`)"
+ else
+ title "upload arvados images SKIPPED because no --upload option set"
+ fi
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+function usage {
+ echo >&2
+ echo >&2 "usage: $0 [options]"
+ echo >&2
+ echo >&2 "$0 options:"
+ echo >&2 " -t, --tags [csv_tags] comma separated tags"
+ echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2
+ echo >&2 " If no options are given, just builds the images."
+}
+
+upload=false
+
+# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+TEMP=`getopt -o hut: \
+ --long help,upload,tags: \
+ -n "$0" -- "$@"`
+
+if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while [ $# -ge 1 ]
+do
+ case $1 in
+ -u | --upload)
+ upload=true
+ shift
+ ;;
+ -t | --tags)
+ case "$2" in
+ "")
+ echo "ERROR: --tags needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ tags=$2;
+ shift 2
+ ;;
+ esac
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+
+EXITCODE=0
+
+COLUMNS=80
+
+title () {
+ printf "\n%*s\n\n" $(((${#title}+$COLUMNS)/2)) "********** $1 **********"
+}
+
+docker_push () {
+ if [[ ! -z "$tags" ]]
+ then
+ for tag in $( echo $tags|tr "," " " )
+ do
+ $DOCKER tag -f $1 $1:$tag
+ done
+ fi
+
+ # Sometimes docker push fails; retry it a few times if necessary.
+ for i in `seq 1 5`; do
+ $DOCKER push $*
+ ECODE=$?
+ if [[ "$ECODE" == "0" ]]; then
+ break
+ fi
+ done
+
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker push $* failed !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+echo $WORKSPACE
+
+# find the docker binary
+DOCKER=`which docker.io`
+
+if [[ "$DOCKER" == "" ]]; then
+ DOCKER=`which docker`
+fi
+
+if [[ "$DOCKER" == "" ]]; then
+ title "Error: you need to have docker installed. Could not find the docker executable."
+ exit 1
+fi
+
+# DOCKER
+title "Starting docker build"
+
+timer_reset
+
+# clean up the docker build environment
+cd "$WORKSPACE"
+cd docker
+rm -f jobs-image
+rm -f config.yml
+
+# Get test config.yml file
+cp $HOME/docker/config.yml .
+
+./build.sh jobs-image
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+title "docker build complete (`timer`)"
+
+title "uploading images"
+
+timer_reset
+
+if [[ "$ECODE" != "0" ]]; then
+ title "upload arvados images SKIPPED because build failed"
+else
+ if [[ $upload == true ]]; then
+ ## 20150526 nico -- *sometimes* dockerhub needs re-login
+ ## even though credentials are already in .dockercfg
+ docker login -u arvados
+
+ docker_push arvados/jobs
+ title "upload arvados images complete (`timer`)"
+ else
+ title "upload arvados images SKIPPED because no --upload option set"
+ fi
+fi
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Orchestrate run-build-packages.sh for every target
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+Options:
+
+--command
+ Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install tests
+--debug
+ Output debug information (default: false)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+set -e
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,test-packages,debug,command:,only-test: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+COMMAND=
+DEBUG=
+TEST_PACKAGES=
+ONLY_TEST=
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --debug)
+ DEBUG="--debug"
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --test-packages)
+ TEST_PACKAGES="--test-packages"
+ ;;
+ --only-test)
+ ONLY_TEST="$1 $2"; shift
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+cd $(dirname $0)
+
+FINAL_EXITCODE=0
+
+for dockerfile_path in $(find -name Dockerfile); do
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ true
+ else
+ FINAL_EXITCODE=$?
+ fi
+done
+
+if test $FINAL_EXITCODE != 0 ; then
+ echo "Build packages failed with code $FINAL_EXITCODE" >&2
+fi
+
+exit $FINAL_EXITCODE
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Orchestrate run-build-packages.sh for one target
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+--target <target>
+ Distribution to build packages for (default: debian7)
+--command
+ Build command to execute (default: use built-in Docker image command)
+--test-packages
+ Run package install test script "test-packages-$target.sh"
+--debug
+ Output debug information (default: false)
+--only-test
+ Test only a specific package
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+set -e
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,debug,test-packages,target:,command:,only-test: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+TARGET=debian7
+COMMAND=
+DEBUG=
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --only-test)
+ packages="$2"; shift
+ ;;
+ --debug)
+ DEBUG=" --debug"
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --test-packages)
+ test_packages=1
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+set -e
+
+if [[ -n "$test_packages" ]]; then
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.rpm)" ]] ; then
+ createrepo $WORKSPACE/packages/$TARGET
+ fi
+
+ if [[ -n "$(find $WORKSPACE/packages/$TARGET -name *.deb)" ]] ; then
+ (cd $WORKSPACE/packages/$TARGET
+ dpkg-scanpackages . 2> >(grep -v 'warning' 1>&2) | gzip -c > Packages.gz
+ )
+ fi
+
+ COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh"
+ IMAGE="arvados/package-test:$TARGET"
+else
+ IMAGE="arvados/build:$TARGET"
+ if [[ "$COMMAND" != "" ]]; then
+ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET$DEBUG"
+ fi
+fi
+
+JENKINS_DIR=$(dirname "$(readlink -e "$0")")
+
+if [[ -n "$test_packages" ]]; then
+ pushd "$JENKINS_DIR/package-test-dockerfiles"
+else
+ pushd "$JENKINS_DIR/package-build-dockerfiles"
+ make "$TARGET/generated"
+fi
+
+echo $TARGET
+cd $TARGET
+time docker build --tag=$IMAGE .
+popd
+
+if test -z "$packages" ; then
+ packages="arvados-api-server
+ arvados-data-manager
+ arvados-docker-cleaner
+ arvados-git-httpd
+ arvados-node-manager
+ arvados-src
+ arvados-workbench
+ crunchstat
+ keepproxy
+ keep-rsync
+ keepstore
+ keep-web
+ libarvados-perl"
+
+ case "$TARGET" in
+ centos6)
+ packages="$packages python27-python-arvados-fuse
+ python27-python-arvados-python-client"
+ ;;
+ *)
+ packages="$packages python-arvados-fuse
+ python-arvados-python-client"
+ ;;
+ esac
+fi
+
+FINAL_EXITCODE=0
+
+package_fails=""
+
+mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET"
+mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET"
+
+docker_volume_args=(
+ -v "$JENKINS_DIR:/jenkins"
+ -v "$WORKSPACE:/arvados"
+ -v /arvados/services/api/vendor/bundle
+ -v /arvados/apps/workbench/vendor/bundle
+ -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache"
+ -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache"
+)
+
+if [[ -n "$test_packages" ]]; then
+ for p in $packages ; do
+ echo
+ echo "START: $p test on $IMAGE" >&2
+ if docker run --rm \
+ "${docker_volume_args[@]}" \
+ --env ARVADOS_DEBUG=1 \
+ --env "TARGET=$TARGET" \
+ --env "WORKSPACE=/arvados" \
+ "$IMAGE" $COMMAND $p
+ then
+ echo "OK: $p test on $IMAGE succeeded" >&2
+ else
+ FINAL_EXITCODE=$?
+ package_fails="$package_fails $p"
+ echo "ERROR: $p test on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
+ fi
+ done
+else
+ echo
+ echo "START: build packages on $IMAGE" >&2
+ if docker run --rm \
+ "${docker_volume_args[@]}" \
+ --env ARVADOS_DEBUG=1 \
+ "$IMAGE" $COMMAND
+ then
+ echo
+ echo "OK: build packages on $IMAGE succeeded" >&2
+ else
+ FINAL_EXITCODE=$?
+ echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2
+ fi
+fi
+
+if test -n "$package_fails" ; then
+ echo "Failed package tests:$package_fails" >&2
+fi
+
+exit $FINAL_EXITCODE
--- /dev/null
+#!/bin/bash
+
+JENKINS_DIR=$(dirname $(readlink -e "$0"))
+. "$JENKINS_DIR/run-library.sh"
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Build Arvados SSO server package
+
+Syntax:
+ WORKSPACE=/path/to/arvados-sso $(basename $0) [options]
+
+Options:
+
+--debug
+ Output debug information (default: false)
+--target
+ Distribution to build packages for (default: debian7)
+
+WORKSPACE=path Path to the Arvados SSO source tree to build packages from
+
+EOF
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+TARGET=debian7
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,build-bundle-packages,debug,target: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --debug)
+ DEBUG=1
+ ;;
+ --test-packages)
+ test_packages=1
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+case "$TARGET" in
+ debian7)
+ FORMAT=deb
+ ;;
+ debian8)
+ FORMAT=deb
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ ;;
+ ubuntu1404)
+ FORMAT=deb
+ ;;
+ centos6)
+ FORMAT=rpm
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+if ! [[ -d "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: $WORKSPACE is not a directory"
+ echo >&2
+ exit 1
+fi
+
+# Test for fpm
+fpm --version >/dev/null 2>&1
+
+if [[ "$?" != 0 ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: fpm not found"
+ echo >&2
+ exit 1
+fi
+
+RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
+RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
+if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
+debug_echo "Workspace is $WORKSPACE"
+
+if [[ -f /etc/profile.d/rvm.sh ]]; then
+ source /etc/profile.d/rvm.sh
+ GEM="rvm-exec default gem"
+else
+ GEM=gem
+fi
+
+# Make all files world-readable -- jenkins runs with umask 027, and has checked
+# out our git tree here
+chmod o+r "$WORKSPACE" -R
+
+# More cleanup - make sure all executables that we'll package are 755
+# No executables in the sso server package
+#find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
+
+# Now fix our umask to something better suited to building and publishing
+# gems and packages
+umask 0022
+
+debug_echo "umask is" `umask`
+
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p "$WORKSPACE/packages/$TARGET"
+fi
+
+# Build the SSO server package
+handle_rails_package arvados-sso-server "$WORKSPACE" \
+ "$WORKSPACE/LICENCE" --url="https://arvados.org" \
+ --description="Arvados SSO server - Arvados is a free and open source platform for big data science." \
+ --license="Expat license"
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+. `dirname "$(readlink -f "$0")"`/run-library.sh
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Build Arvados packages
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0) [options]
+
+Options:
+
+--build-bundle-packages (default: false)
+ Build api server and workbench packages with vendor/bundle included
+--debug
+ Output debug information (default: false)
+--target
+ Distribution to build packages for (default: debian7)
+--command
+ Build command to execute (defaults to the run command defined in the
+ Docker image)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+
+EOF
+
+EXITCODE=0
+DEBUG=${ARVADOS_DEBUG:-0}
+TARGET=debian7
+COMMAND=
+
+PARSEDOPTS=$(getopt --name "$0" --longoptions \
+ help,build-bundle-packages,debug,target: \
+ -- "" "$@")
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+eval set -- "$PARSEDOPTS"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --target)
+ TARGET="$2"; shift
+ ;;
+ --debug)
+ DEBUG=1
+ ;;
+ --command)
+ COMMAND="$2"; shift
+ ;;
+ --)
+ if [ $# -gt 1 ]; then
+ echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
+ exit 1
+ fi
+ ;;
+ esac
+ shift
+done
+
+if [[ "$COMMAND" != "" ]]; then
+ COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
+fi
+
+STDOUT_IF_DEBUG=/dev/null
+STDERR_IF_DEBUG=/dev/null
+DASHQ_UNLESS_DEBUG=-q
+if [[ "$DEBUG" != 0 ]]; then
+ STDOUT_IF_DEBUG=/dev/stdout
+ STDERR_IF_DEBUG=/dev/stderr
+ DASHQ_UNLESS_DEBUG=
+fi
+
+declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
+
+PYTHON2_VERSION=2.7
+PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+
+case "$TARGET" in
+ debian7)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ 'pycurl<7.21.5' contextlib2)
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ debian8)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ 'pycurl<7.21.5')
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ ubuntu1204)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
+ contextlib2 \
+ 'pycurl<7.21.5')
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ ;;
+ ubuntu1404)
+ FORMAT=deb
+ PYTHON2_PACKAGE=python$PYTHON2_VERSION
+ PYTHON2_PKG_PREFIX=python
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_PKG_PREFIX=python3
+ PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
+ google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
+ rsa 'pycurl<7.21.5' backports.ssl_match_hostname)
+ PYTHON3_BACKPORTS=(docker-py requests websocket-client)
+ ;;
+ centos6)
+ FORMAT=rpm
+ PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+ PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+ PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+ PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
+ python-daemon lockfile llfuse==0.41.1 'pbr<1.0')
+ PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+ export PYCURL_SSL_LIBRARY=nss
+ ;;
+ *)
+ echo -e "$0: Unknown target '$TARGET'.\n" >&2
+ exit 1
+ ;;
+esac
+
+
+if ! [[ -n "$WORKSPACE" ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+# Test for fpm
+fpm --version >/dev/null 2>&1
+
+if [[ "$?" != 0 ]]; then
+ echo >&2 "$helpmessage"
+ echo >&2
+ echo >&2 "Error: fpm not found"
+ echo >&2
+ exit 1
+fi
+
+EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
+EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
+
+RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
+RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
+if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+fi
+
+debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
+debug_echo "Workspace is $WORKSPACE"
+
+if [[ -f /etc/profile.d/rvm.sh ]]; then
+ source /etc/profile.d/rvm.sh
+ GEM="rvm-exec default gem"
+else
+ GEM=gem
+fi
+
+# Make all files world-readable -- jenkins runs with umask 027, and has checked
+# out our git tree here
+chmod o+r "$WORKSPACE" -R
+
+# More cleanup - make sure all executables that we'll package are 755
+find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
+
+# Now fix our umask to something better suited to building and publishing
+# gems and packages
+umask 0022
+
+debug_echo "umask is" `umask`
+
+if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
+ mkdir -p $WORKSPACE/packages/$TARGET
+fi
+
+# Perl packages
+debug_echo -e "\nPerl packages\n"
+
+cd "$WORKSPACE/sdk/perl"
+
+if [[ -e Makefile ]]; then
+ make realclean >"$STDOUT_IF_DEBUG"
+fi
+find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
+ -delete
+rm -rf install
+
+perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
+ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
+ fpm_build install/lib/=/usr/share libarvados-perl \
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+ "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+
+# Ruby gems
+debug_echo -e "\nRuby gems\n"
+
+FPM_GEM_PREFIX=$($GEM environment gemdir)
+
+cd "$WORKSPACE/sdk/ruby"
+handle_ruby_gem arvados
+
+cd "$WORKSPACE/sdk/cli"
+handle_ruby_gem arvados-cli
+
+cd "$WORKSPACE/services/login-sync"
+handle_ruby_gem arvados-login-sync
+
+# Python packages
+debug_echo -e "\nPython packages\n"
+
+cd "$WORKSPACE/sdk/pam"
+handle_python_package
+
+cd "$WORKSPACE/sdk/python"
+handle_python_package
+
+cd "$WORKSPACE/sdk/cwl"
+handle_python_package
+
+cd "$WORKSPACE/services/fuse"
+handle_python_package
+
+cd "$WORKSPACE/services/nodemanager"
+handle_python_package
+
+# arvados-src
+(
+ set -e
+
+ cd "$WORKSPACE"
+ COMMIT_HASH=$(format_last_commit_here "%H")
+
+ SRC_BUILD_DIR=$(mktemp -d)
+ # mktemp creates the directory with 0700 permissions by default
+ chmod 755 $SRC_BUILD_DIR
+ git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+ cd "$SRC_BUILD_DIR"
+
+ # go into detached-head state
+ git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+ echo "$COMMIT_HASH" >git-commit.version
+
+ cd "$SRC_BUILD_DIR"
+ PKG_VERSION=$(version_from_git)
+ cd $WORKSPACE/packages/$TARGET
+ fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+ rm -rf "$SRC_BUILD_DIR"
+)
+
+# On older platforms we need to publish a backport of libfuse >=2.9.2,
+# and we need to build and install it here in order to even build an
+# llfuse package.
+cd $WORKSPACE/packages/$TARGET
+if [[ $TARGET =~ ubuntu1204 ]]; then
+ # port libfuse 2.9.2 to Ubuntu 12.04
+ LIBFUSE_DIR=$(mktemp -d)
+ (
+ cd $LIBFUSE_DIR
+ # download fuse 2.9.2 ubuntu 14.04 source package
+ file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+ file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+ file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}"
+
+ # install dpkg-source and dpkg-buildpackage commands
+ apt-get install -y --no-install-recommends dpkg-dev
+
+ # extract source and apply patches
+ dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc
+ rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc
+
+ # add new version to changelog
+ cd fuse-2.9.2
+ (
+ echo "fuse (2.9.2-5) precise; urgency=low"
+ echo
+ echo " * Backported from trusty-security to precise"
+ echo
+ echo " -- Joshua Randall <jcrandall@alum.mit.edu> Thu, 4 Feb 2016 11:31:00 -0000"
+ echo
+ cat debian/changelog
+ ) > debian/changelog.new
+ mv debian/changelog.new debian/changelog
+
+ # install build-deps and build
+ apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev
+ dpkg-buildpackage -rfakeroot -b
+ )
+ fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5
+ dpkg -i \
+ "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \
+ "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \
+ "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb"
+ apt-get -y --no-install-recommends -f install
+ rm -rf $LIBFUSE_DIR
+elif [[ $TARGET =~ centos6 ]]; then
+ # port fuse 2.9.2 to centos 6
+ # install tools to build rpm from source
+ yum install -y rpm-build redhat-rpm-config
+ LIBFUSE_DIR=$(mktemp -d)
+ (
+ cd "$LIBFUSE_DIR"
+ # download fuse 2.9.2 centos 7 source rpm
+ file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}"
+ (
+ # modify source rpm spec to remove conflict on filesystem version
+ mkdir -p /root/rpmbuild/SOURCES
+ cd /root/rpmbuild/SOURCES
+ rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
+ perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
+ )
+ # build rpms from source
+ rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
+ rm -f fuse-2.9.2-6.el7.src.rpm
+ # move built RPMs to LIBFUSE_DIR
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/
+ rm -rf /root/rpmbuild
+ )
+ fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5
+ fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+ fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends
+ yum install -y \
+ "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \
+ "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \
+ "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm"
+fi
+
+# Go binaries
+cd $WORKSPACE/packages/$TARGET
+export GOPATH=$(mktemp -d)
+package_go_binary services/keepstore keepstore \
+ "Keep storage daemon, accessible to clients on the LAN"
+package_go_binary services/keepproxy keepproxy \
+ "Make a Keep cluster accessible to clients that are not on the LAN"
+package_go_binary services/keep-web keep-web \
+ "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary services/datamanager arvados-data-manager \
+ "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary services/arv-git-httpd arvados-git-httpd \
+ "Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/crunchstat crunchstat \
+ "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary tools/keep-rsync keep-rsync \
+ "Copy all data from one set of Keep servers to another"
+
+# The Python SDK
+# Please resist the temptation to add --no-python-fix-name to the fpm call here
+# (which would remove the python- prefix from the package name), because this
+# package is a dependency of arvados-fuse, and fpm can not omit the python-
+# prefix from only one of the dependencies of a package... Maybe I could
+# whip up a patch and send it upstream, but that will be for another day. Ward,
+# 2014-05-15
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/sdk/python/build"
+fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
+
+# cwl-runner
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/sdk/cwl/build"
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
+
+# The PAM module
+if [[ $TARGET =~ debian|ubuntu ]]; then
+ cd $WORKSPACE/packages/$TARGET
+ rm -rf "$WORKSPACE/sdk/pam/build"
+ fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
+fi
+
+# The FUSE driver
+# Please see comment about --no-python-fix-name above; we stay consistent and do
+# not omit the python- prefix first.
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/fuse/build"
+fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+
+# The node manager
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/nodemanager/build"
+fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+
+# The Docker image cleaner
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/services/dockercleaner/build"
+fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+
+# Forked libcloud
+LIBCLOUD_DIR=$(mktemp -d)
+(
+ cd $LIBCLOUD_DIR
+ git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
+ git checkout apache-libcloud-$LIBCLOUD_PIN
+ # libcloud is absurdly noisy without -q, so force -q here
+ OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
+ DASHQ_UNLESS_DEBUG=-q
+ handle_python_package
+ DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
+)
+fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+rm -rf $LIBCLOUD_DIR
+
+# Python 2 dependencies
+declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
+# Add --no-use-wheel if this pip knows it.
+pip wheel --help >/dev/null 2>&1
+case "$?" in
+ 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
+ 2) ;;
+ *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
+esac
+
+for deppkg in "${PYTHON_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+ case "$deppkg" in
+ httplib2|google-api-python-client)
+ # Work around 0640 permissions on some package files.
+ # See #7591 and #7991.
+ pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
+ set -e
+ cd "$pyfpm_workdir"
+ pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+ tar -xf "$deppkg"-*.tar*
+ cd "$deppkg"-*/
+ "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
+ chmod -R go+rX .
+ set +e
+ # --iteration 2 provides an upgrade for previously built
+ # buggy packages.
+ fpm_build . "$outname" "" python "" --iteration 2
+ # The upload step uses the package timestamp to determine
+ # whether it's new. --no-clobber plays nice with that.
+ mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
+ )
+ if [ 0 != "$?" ]; then
+ echo "ERROR: $deppkg build process failed"
+ EXITCODE=1
+ fi
+ if [ -n "$pyfpm_workdir" ]; then
+ rm -rf "$pyfpm_workdir"
+ fi
+ ;;
+ *)
+ fpm_build "$deppkg" "$outname"
+ ;;
+ esac
+done
+
+# Python 3 dependencies
+for deppkg in "${PYTHON3_BACKPORTS[@]}"; do
+ outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/")
+ # The empty string is the vendor argument: these aren't Curoverse software.
+ fpm_build "$deppkg" "$outname" "" python3
+done
+
+# Build the API server package
+handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados API server - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0"
+
+# Build the workbench server package
+(
+ set -e
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up application.yml and production.rb so that asset precompilation works
+ \cp config/application.yml.example config/application.yml -f
+ \cp config/environments/production.rb.example config/environments/production.rb -f
+ sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+
+ RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+
+ # Remove generated configuration files so they don't go in the package.
+ rm config/application.yml config/environments/production.rb
+)
+
+if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0"
+fi
+
+# clean up temporary GOPATH
+rm -rf "$GOPATH"
+
+exit $EXITCODE
--- /dev/null
+#!/bin/bash
+
+# A library of functions shared by the various scripts in this directory.
+
+# This is the timestamp about when we merged changed to include licenses
+# with Arvados packages. We use it as a heuristic to add revisions for
+# older packages.
+LICENSE_PACKAGE_TS=20151208015500
+
+debug_echo () {
+ echo "$@" >"$STDOUT_IF_DEBUG"
+}
+
+find_easy_install() {
+ for version_suffix in "$@"; do
+ if "easy_install$version_suffix" --version >/dev/null 2>&1; then
+ echo "easy_install$version_suffix"
+ return 0
+ fi
+ done
+ cat >&2 <<EOF
+$helpmessage
+
+Error: easy_install$1 (from Python setuptools module) not found
+
+EOF
+ exit 1
+}
+
+format_last_commit_here() {
+ local format="$1"; shift
+ TZ=UTC git log -n1 --first-parent "--format=format:$format" .
+}
+
+version_from_git() {
+ # Generates a version number from the git log for the current working
+ # directory, and writes it to stdout.
+ local git_ts git_hash
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+}
+
+nohash_version_from_git() {
+ version_from_git | cut -d. -f1-3
+}
+
+timestamp_from_git() {
+ format_last_commit_here "%ct"
+}
+
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ # This package doesn't need rebuilding.
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
+handle_ruby_gem() {
+ local gem_name="$1"; shift
+ local gem_version="$(nohash_version_from_git)"
+ local gem_src_dir="$(pwd)"
+
+ if ! [[ -e "${gem_name}-${gem_version}.gem" ]]; then
+ find -maxdepth 1 -name "${gem_name}-*.gem" -delete
+
+ # -q appears to be broken in gem version 2.2.2
+ $GEM build "$gem_name.gemspec" $DASHQ_UNLESS_DEBUG >"$STDOUT_IF_DEBUG" 2>"$STDERR_IF_DEBUG"
+ fi
+}
+
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
+package_go_binary() {
+ local src_path="$1"; shift
+ local prog="$1"; shift
+ local description="$1"; shift
+ local license_file="${1:-agpl-3.0.txt}"; shift
+
+ debug_echo "package_go_binary $src_path as $prog"
+
+ local basename="${src_path##*/}"
+
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
+ local version="$(version_from_git)"
+ local timestamp="$(timestamp_from_git)"
+
+ # If the command imports anything from the Arvados SDK, bump the
+ # version number and build a new package whenever the SDK changes.
+ if grep -qr git.curoverse.com/arvados .; then
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
+ if [[ $(timestamp_from_git) -gt "$timestamp" ]]; then
+ version=$(version_from_git)
+ fi
+ fi
+
+ cd $WORKSPACE/packages/$TARGET
+ go get "git.curoverse.com/arvados.git/$src_path"
+ fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file"
+}
+
+default_iteration() {
+ local package_name="$1"; shift
+ local package_version="$1"; shift
+ local iteration=1
+ if [[ $package_version =~ ^0\.1\.([0-9]{14})(\.|$) ]] && \
+ [[ ${BASH_REMATCH[1]} -le $LICENSE_PACKAGE_TS ]]; then
+ iteration=2
+ fi
+ echo $iteration
+}
+
+_build_rails_package_scripts() {
+ local pkgname="$1"; shift
+ local destdir="$1"; shift
+ local srcdir="$RUN_BUILD_PACKAGES_PATH/rails-package-scripts"
+ for scriptname in postinst prerm postrm; do
+ cat "$srcdir/$pkgname.sh" "$srcdir/step2.sh" "$srcdir/$scriptname.sh" \
+ >"$destdir/$scriptname" || return $?
+ done
+}
+
+handle_rails_package() {
+ local pkgname="$1"; shift
+ local srcdir="$1"; shift
+ local license_path="$1"; shift
+ local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
+ local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ set -e
+ _build_rails_package_scripts "$pkgname" "$scripts_dir"
+ cd "$srcdir"
+ mkdir -p tmp
+ version_from_git >"$version_file"
+ git rev-parse HEAD >git-commit.version
+ bundle package --all
+ )
+ if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
+ echo "ERROR: $pkgname package prep failed" >&2
+ rm -rf "$scripts_dir" "$version_file"
+ EXITCODE=1
+ return 1
+ fi
+ local railsdir="/var/www/${pkgname%-server}/current"
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
+ "$(cat "$version_file")")
+ local license_arg="$license_path=$railsdir/$(basename "$license_path")"
+ # --iteration=5 accommodates the package script bugfixes #8371 and #8413.
+ local -a switches=(--iteration=5
+ --after-install "$scripts_dir/postinst"
+ --before-remove "$scripts_dir/prerm"
+ --after-remove "$scripts_dir/postrm")
+ # For some reason fpm excludes need to not start with /.
+ local exclude_root="${railsdir#/}"
+ # .git and packages are for the SSO server, which is built from its
+ # repository root.
+ local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+ config/deploy\* config/application.yml)
+ # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
+ if [[ "$pkgname" != "arvados-workbench" ]]; then
+ exclude_list+=('config/database.yml')
+ fi
+ for exclude in ${exclude_list[@]}; do
+ switches+=(-x "$exclude_root/$exclude")
+ done
+ fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
+ rm -rf "$scripts_dir" "$version_file"
+}
+
+# Build packages for everything
+fpm_build () {
+ # The package source. Depending on the source type, this can be a
+ # path, or the name of the package in an upstream repository (e.g.,
+ # pip).
+ PACKAGE=$1
+ shift
+ # The name of the package to build. Defaults to $PACKAGE.
+ PACKAGE_NAME=${1:-$PACKAGE}
+ shift
+ # Optional: the vendor of the package. Should be "Curoverse, Inc." for
+ # packages of our own software. Passed to fpm --vendor.
+ VENDOR=$1
+ shift
+ # The type of source package. Passed to fpm -s. Default "python".
+ PACKAGE_TYPE=${1:-python}
+ shift
+ # Optional: the package version number. Passed to fpm -v.
+ VERSION=$1
+ shift
+
+ case "$PACKAGE_TYPE" in
+ python)
+ # All Arvados Python2 packages depend on Python 2.7.
+ # Make sure we build with that for consistency.
+ set -- "$@" --python-bin python2.7 \
+ --python-easyinstall "$EASY_INSTALL2" \
+ --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
+ --depends "$PYTHON2_PACKAGE"
+ ;;
+ python3)
+ # fpm does not actually support a python3 package type. Instead
+ # we recognize it as a convenience shortcut to add several
+ # necessary arguments to fpm's command line later, after we're
+ # done handling positional arguments.
+ PACKAGE_TYPE=python
+ set -- "$@" --python-bin python3 \
+ --python-easyinstall "$EASY_INSTALL3" \
+ --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
+ --depends "$PYTHON3_PACKAGE"
+ ;;
+ esac
+
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ if [ python = "$PACKAGE_TYPE" ]; then
+ COMMAND_ARR+=(--exclude=\*/{dist,site}-packages/tests/\*)
+ if [ deb = "$FORMAT" ]; then
+ # Dependencies are built from setup.py. Since setup.py will never
+ # refer to Debian package iterations, it doesn't make sense to
+ # enforce those in the .deb dependencies.
+ COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
+ fi
+ fi
+
+ if [[ "${DEBUG:-0}" != "0" ]]; then
+ COMMAND_ARR+=('--verbose' '--log' 'info')
+ fi
+
+ if [[ "$PACKAGE_NAME" != "$PACKAGE" ]]; then
+ COMMAND_ARR+=('-n' "$PACKAGE_NAME")
+ fi
+
+ if [[ "$VENDOR" != "" ]]; then
+ COMMAND_ARR+=('--vendor' "$VENDOR")
+ fi
+
+ if [[ "$VERSION" != "" ]]; then
+ COMMAND_ARR+=('-v' "$VERSION")
+ fi
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$(default_iteration "$PACKAGE" "$VERSION")")
+
+ # Append --depends X and other arguments specified by fpm-info.sh in
+ # the package source dir. These are added last so they can override
+ # the arguments added by this script.
+ declare -a fpm_args=()
+ declare -a build_depends=()
+ declare -a fpm_depends=()
+ declare -a fpm_exclude=()
+ declare -a fpm_dirs=(
+ # source dir part of 'dir' package ("/source=/dest" => "/source"):
+ "${PACKAGE%%=/*}"
+ # backports ("llfuse==0.41.1" => "backports/python-llfuse")
+ "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
+ for pkgdir in "${fpm_dirs[@]}"; do
+ fpminfo="$pkgdir/fpm-info.sh"
+ if [[ -e "$fpminfo" ]]; then
+ debug_echo "Loading fpm overrides from $fpminfo"
+ source "$fpminfo"
+ break
+ fi
+ done
+ for pkg in "${build_depends[@]}"; do
+ if [[ $TARGET =~ debian|ubuntu ]]; then
+ pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
+ if [[ -e $pkg_deb ]]; then
+ echo "Installing build_dep $pkg from $pkg_deb"
+ dpkg -i "$pkg_deb"
+ else
+ echo "Attemping to install build_dep $pkg using apt-get"
+ apt-get install -y "$pkg"
+ fi
+ apt-get -y -f install
+ else
+ pkg_rpm=$(ls "$WORKSPACE/packages/$TARGET/$pkg"-[0-9]*.rpm | sort -rg | awk 'NR==1')
+ if [[ -e $pkg_rpm ]]; then
+ echo "Installing build_dep $pkg from $pkg_rpm"
+ rpm -i "$pkg_rpm"
+ else
+ echo "Attemping to install build_dep $pkg"
+ rpm -i "$pkg"
+ fi
+ fi
+ done
+ for i in "${fpm_depends[@]}"; do
+ COMMAND_ARR+=('--depends' "$i")
+ done
+ for i in "${fpm_exclude[@]}"; do
+ COMMAND_ARR+=('--exclude' "$i")
+ done
+
+ # Append remaining function arguments directly to fpm's command line.
+ for i; do
+ COMMAND_ARR+=("$i")
+ done
+
+ COMMAND_ARR+=("${fpm_args[@]}")
+
+ COMMAND_ARR+=("$PACKAGE")
+
+ debug_echo -e "\n${COMMAND_ARR[@]}\n"
+
+ FPM_RESULTS=$("${COMMAND_ARR[@]}")
+ FPM_EXIT_CODE=$?
+
+ fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
+}
+
+# verify build results
+fpm_verify () {
+ FPM_EXIT_CODE=$1
+ shift
+ FPM_RESULTS=$@
+
+ FPM_PACKAGE_NAME=''
+ if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
+ FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
+ fi
+
+ if [[ "$FPM_PACKAGE_NAME" == "" ]]; then
+ EXITCODE=1
+ echo "Error: $PACKAGE: Unable to figure out package name from fpm results:"
+ echo
+ echo $FPM_RESULTS
+ echo
+ elif [[ "$FPM_RESULTS" =~ "File already exists" ]]; then
+ echo "Package $FPM_PACKAGE_NAME exists, not rebuilding"
+ elif [[ 0 -ne "$FPM_EXIT_CODE" ]]; then
+ echo "Error building package for $1:\n $FPM_RESULTS"
+ fi
+}
+
+install_package() {
+ PACKAGES=$@
+ if [[ "$FORMAT" == "deb" ]]; then
+ $SUDO apt-get install $PACKAGES --yes
+ elif [[ "$FORMAT" == "rpm" ]]; then
+ $SUDO yum -q -y install $PACKAGES
+ fi
+}
--- /dev/null
+#!/bin/bash
+
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): Install and test Arvados components.
+
+Exit non-zero if any tests fail.
+
+Syntax:
+ $(basename $0) WORKSPACE=/path/to/arvados [options]
+
+Options:
+
+--skip FOO Do not test the FOO component.
+--only FOO Do not test anything except the FOO component.
+--temp DIR Install components and dependencies under DIR instead of
+ making a new temporary directory. Implies --leave-temp.
+--leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
+ Instead, show the path to give as --temp to reuse them in
+ subsequent invocations.
+--skip-install Do not run any install steps. Just run tests.
+ You should provide GOPATH, GEMHOME, and VENVDIR options
+ from a previous invocation if you use this option.
+--only-install Run specific install step
+WORKSPACE=path Arvados source tree to test.
+CONFIGSRC=path Dir with api server config files to copy into source tree.
+ (If none given, leave config files alone in source tree.)
+services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
+ Restrict apiserver tests to the given file
+sdk/python_test="--test-suite test.test_keep_locator"
+ Restrict Python SDK tests to the given class
+apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
+ Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+ Show all log messages, even when tests pass (also works
+ with services/keepstore_test etc.)
+ARVADOS_DEBUG=1
+ Print more debug messages
+envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
+ *_test, and other examples shown above.
+
+Assuming --skip-install is not given, all components are installed
+into \$GOPATH, \$VENDIR, and \$GEMHOME before running any tests. Many
+test suites depend on other components being installed, and installing
+everything tends to be quicker than debugging dependencies.
+
+As a special concession to the current CI server config, CONFIGSRC
+defaults to $HOME/arvados-api-server if that directory exists.
+
+More information and background:
+
+https://arvados.org/projects/arvados/wiki/Running_tests
+
+Available tests:
+
+apps/workbench
+apps/workbench_benchmark
+apps/workbench_profile
+doc
+services/api
+services/arv-git-httpd
+services/crunchstat
+services/dockercleaner
+services/fuse
+services/keep-web
+services/keepproxy
+services/keepstore
+services/login-sync
+services/nodemanager
+services/crunch-run
+services/crunch-dispatch-local
+services/crunch-dispatch-slurm
+sdk/cli
+sdk/pam
+sdk/python
+sdk/ruby
+sdk/go/arvadosclient
+sdk/go/keepclient
+sdk/go/manifest
+sdk/go/blockdigest
+sdk/go/streamer
+sdk/go/crunchrunner
+sdk/cwl
+tools/crunchstat-summary
+tools/keep-rsync
+
+EOF
+
+# First make sure to remove any ARVADOS_ variables from the calling
+# environment that could interfere with the tests.
+unset $(env | cut -d= -f1 | grep \^ARVADOS_)
+
+# Reset other variables that could affect our [tests'] behavior by
+# accident.
+GITDIR=
+GOPATH=
+VENVDIR=
+VENV3DIR=
+PYTHONPATH=
+GEMHOME=
+PERLINSTALLBASE=
+
+COLUMNS=80
+
+skip_install=
+temp=
+temp_preserve=
+
+clear_temp() {
+ if [[ -z "$temp" ]]; then
+ # we didn't even get as far as making a temp dir
+ :
+ elif [[ -z "$temp_preserve" ]]; then
+ rm -rf "$temp"
+ else
+ echo "Leaving behind temp dirs in $temp"
+ fi
+}
+
+fatal() {
+ clear_temp
+ echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
+ exit 1
+}
+
+report_outcomes() {
+ for x in "${successes[@]}"
+ do
+ echo "Pass: $x"
+ done
+
+ if [[ ${#failures[@]} == 0 ]]
+ then
+ echo "All test suites passed."
+ else
+ echo "Failures (${#failures[@]}):"
+ for x in "${failures[@]}"
+ do
+ echo "Fail: $x"
+ done
+ fi
+}
+
+exit_cleanly() {
+ trap - INT
+ create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
+ rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+ stop_services
+ rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+ report_outcomes
+ clear_temp
+ exit ${#failures}
+}
+
+sanity_checks() {
+ ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ echo Checking dependencies:
+ echo -n 'virtualenv: '
+ virtualenv --version \
+ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
+ echo -n 'go: '
+ go version \
+ || fatal "No go binary. See http://golang.org/doc/install"
+ echo -n 'gcc: '
+ gcc --version | egrep ^gcc \
+ || fatal "No gcc. Try: apt-get install build-essential"
+ echo -n 'fuse.h: '
+ find /usr/include -wholename '*fuse/fuse.h' \
+ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'pyconfig.h: '
+ find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+ || fatal "No pyconfig.h. Try: apt-get install python-dev"
+ echo -n 'nginx: '
+ PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+ || fatal "No nginx. Try: apt-get install nginx"
+ echo -n 'perl: '
+ perl -v | grep version \
+ || fatal "No perl. Try: apt-get install perl"
+ for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
+ echo -n "perl $mod: "
+ perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
+ || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
+ done
+ echo -n 'gitolite: '
+ which gitolite \
+ || fatal "No gitolite. Try: apt-get install gitolite3"
+}
+
+rotate_logfile() {
+ # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
+ # $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
+ if [[ -f "$1/$2" ]]; then
+ THEDATE=`date +%Y%m%d%H%M%S`
+ mv "$1/$2" "$1/$THEDATE-$BUILD_NUMBER-$2"
+ gzip "$1/$THEDATE-$BUILD_NUMBER-$2"
+ fi
+}
+
+declare -a failures
+declare -A skip
+declare -A testargs
+skip[apps/workbench_profile]=1
+
+while [[ -n "$1" ]]
+do
+ arg="$1"; shift
+ case "$arg" in
+ --help)
+ echo >&2 "$helpmessage"
+ echo >&2
+ exit 1
+ ;;
+ --skip)
+ skipwhat="$1"; shift
+ skip[$skipwhat]=1
+ ;;
+ --only)
+ only="$1"; skip[$1]=""; shift
+ ;;
+ --skip-install)
+ skip_install=1
+ ;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
+ --temp)
+ temp="$1"; shift
+ temp_preserve=1
+ ;;
+ --leave-temp)
+ temp_preserve=1
+ ;;
+ --retry)
+ retry=1
+ ;;
+ *_test=*)
+ suite="${arg%%_test=*}"
+ args="${arg#*=}"
+ testargs["$suite"]="$args"
+ ;;
+ *=*)
+ eval export $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
+ ;;
+ *)
+ echo >&2 "$0: Unrecognized option: '$arg'. Try: $0 --help"
+ exit 1
+ ;;
+ esac
+done
+
+start_api() {
+ echo 'Starting API server...'
+ cd "$WORKSPACE" \
+ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
+ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
+ && export ARVADOS_TEST_API_INSTALLED="$$" \
+ && (env | egrep ^ARVADOS)
+}
+
+start_nginx_proxy_services() {
+ echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && python sdk/python/tests/run_test_server.py start_keep-web \
+ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py start_nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1
+}
+
+stop_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ unset ARVADOS_TEST_PROXY_SERVICES
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy
+ fi
+ if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
+ unset ARVADOS_TEST_API_HOST
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop
+ fi
+}
+
+interrupt() {
+ failures+=("($(basename $0) interrupted)")
+ exit_cleanly
+}
+trap interrupt INT
+
+sanity_checks
+
+echo "WORKSPACE=$WORKSPACE"
+
+if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
+ # Jenkins expects us to use this by default.
+ CONFIGSRC="$HOME/arvados-api-server"
+fi
+
+# Clean up .pyc files that may exist in the workspace
+cd "$WORKSPACE"
+find -name '*.pyc' -delete
+
+if [[ -z "$temp" ]]; then
+ temp="$(mktemp -d)"
+fi
+
+# Set up temporary install dirs (unless existing dirs were supplied)
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
+do
+ if [[ -z "${!tmpdir}" ]]; then
+ eval "$tmpdir"="$temp/$tmpdir"
+ fi
+ if ! [[ -d "${!tmpdir}" ]]; then
+ mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)"
+ fi
+done
+
+setup_ruby_environment() {
+ if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
+ source "$HOME/.rvm/scripts/rvm"
+ using_rvm=true
+ elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
+ source "/usr/local/rvm/scripts/rvm"
+ using_rvm=true
+ else
+ using_rvm=false
+ fi
+
+ if [[ "$using_rvm" == true ]]; then
+ # If rvm is in use, we can't just put separate "dependencies"
+ # and "gems-under-test" paths to GEM_PATH: passenger resets
+ # the environment to the "current gemset", which would lose
+ # our GEM_PATH and prevent our test suites from running ruby
+ # programs (for example, the Workbench test suite could not
+ # boot an API server or run arv). Instead, we have to make an
+ # rvm gemset and use it for everything.
+
+ [[ `type rvm | head -n1` == "rvm is a function" ]] \
+ || fatal 'rvm check'
+
+ # Put rvm's favorite path back in first place (overriding
+ # virtualenv, which just put itself there). Ignore rvm's
+ # complaint about not being in first place already.
+ rvm use @default 2>/dev/null
+
+ # Create (if needed) and switch to an @arvados-tests
+ # gemset. (Leave the choice of ruby to the caller.)
+ rvm use @arvados-tests --create \
+ || fatal 'rvm gemset setup'
+
+ rvm env
+ else
+ # When our "bundle install"s need to install new gems to
+ # satisfy dependencies, we want them to go where "gem install
+ # --user-install" would put them. (However, if the caller has
+ # already set GEM_HOME, we assume that's where dependencies
+ # should be installed, and we should leave it alone.)
+
+ if [ -z "$GEM_HOME" ]; then
+ user_gempath="$(gem env gempath)"
+ export GEM_HOME="${user_gempath%%:*}"
+ fi
+ PATH="$(gem env gemdir)/bin:$PATH"
+
+ # When we build and install our own gems, we install them in our
+ # $GEMHOME tmpdir, and we want them to be at the front of GEM_PATH and
+ # PATH so integration tests prefer them over other versions that
+ # happen to be installed in $user_gempath, system dirs, etc.
+
+ tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
+ PATH="$tmpdir_gem_home/bin:$PATH"
+ export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
+
+ echo "Will install dependencies to $(gem env gemdir)"
+ echo "Will install arvados gems to $tmpdir_gem_home"
+ echo "Gem search path is GEM_PATH=$GEM_PATH"
+ fi
+}
+
+with_test_gemset() {
+ if [[ "$using_rvm" == true ]]; then
+ "$@"
+ else
+ GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
+ fi
+}
+
+gem_uninstall_if_exists() {
+ if gem list "$1\$" | egrep '^\w'; then
+ gem uninstall --force --all --executables "$1"
+ fi
+}
+
+setup_virtualenv() {
+ local venvdest="$1"; shift
+ if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
+ virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+ fi
+ "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7'
+ # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
+ "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
+}
+
+export PERLINSTALLBASE
+export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+
+export GOPATH
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+ || fatal "symlink failed"
+
+setup_virtualenv "$VENVDIR" --python python2.7
+. "$VENVDIR/bin/activate"
+
+# Needed for run_test_server.py which is used by certain (non-Python) tests.
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+ || pip install PyYAML >/dev/null \
+ || fatal "pip install PyYAML failed"
+
+# Preinstall forked version of libcloud, because nodemanager "pip install"
+# won't pick it up by default.
+pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
+ || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
+ || fatal "pip install apache-libcloud failed"
+
+# This will help people who reuse --temp dirs when we upgrade to llfuse 0.42
+if egrep -q 'llfuse.*>= *0\.42' "$WORKSPACE/services/fuse/setup.py"; then
+ # Uninstall old llfuse, because services/fuse "pip install" won't
+ # upgrade it by default.
+ if pip freeze | egrep '^llfuse==0\.41\.'; then
+ yes | pip uninstall 'llfuse<0.42'
+ fi
+fi
+
+# Deactivate Python 2 virtualenv
+deactivate
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ setup_virtualenv "$VENV3DIR" --python python3
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
+
+# Reactivate Python 2 virtualenv
+. "$VENVDIR/bin/activate"
+
+# Note: this must be the last time we change PATH, otherwise rvm will
+# whine a lot.
+setup_ruby_environment
+
+echo "PATH is $PATH"
+
+if ! which bundler >/dev/null
+then
+ gem install --user-install bundler || fatal 'Could not install bundler'
+fi
+
+checkexit() {
+ if [[ "$1" != "0" ]]; then
+ title "!!!!!! $2 FAILED !!!!!!"
+ failures+=("$2 (`timer`)")
+ else
+ successes+=("$2 (`timer`)")
+ fi
+}
+
+timer_reset() {
+ t0=$SECONDS
+}
+
+timer() {
+ echo -n "$(($SECONDS - $t0))s"
+}
+
+retry() {
+ while ! ${@} && [[ "$retry" == 1 ]]
+ do
+ read -p 'Try again? [Y/n] ' x
+ if [[ "$x" != "y" ]] && [[ "$x" != "" ]]
+ then
+ break
+ fi
+ done
+}
+
+do_test() {
+ retry do_test_once ${@}
+}
+
+do_test_once() {
+ unset result
+ if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
+ then
+ title "Running $1 tests"
+ timer_reset
+ if [[ "$2" == "go" ]]
+ then
+ covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
+ coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ # We do "go get -t" here to catch compilation errors
+ # before trying "go test". Otherwise, coverage-reporting
+ # mode makes Go show the wrong line numbers when reporting
+ # compilation errors.
+ if [[ -n "${testargs[$1]}" ]]
+ then
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && \
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} ${testargs[$1]}
+ else
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+ fi
+ result="$?"
+ go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
+ rm "$WORKSPACE/tmp/.$covername.tmp"
+ elif [[ "$2" == "pip" ]]
+ then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
+ elif [[ "$2" != "" ]]
+ then
+ "test_$2"
+ else
+ "test_$1"
+ fi
+ result=${result:-$?}
+ checkexit $result "$1 tests"
+ title "End of $1 tests (`timer`)"
+ return $result
+ else
+ title "Skipping $1 tests"
+ fi
+}
+
+do_install() {
+ retry do_install_once ${@}
+}
+
+do_install_once() {
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
+ then
+ title "Running $1 install"
+ timer_reset
+ if [[ "$2" == "go" ]]
+ then
+ go get -t "git.curoverse.com/arvados.git/$1"
+ elif [[ "$2" == "pip" ]]
+ then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it believes the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually installed the local package
+ # we just built.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
+ elif [[ "$2" != "" ]]
+ then
+ "install_$2"
+ else
+ "install_$1"
+ fi
+ result=$?
+ checkexit $result "$1 install"
+ title "End of $1 install (`timer`)"
+ return $result
+ else
+ title "Skipping $1 install"
+ fi
+}
+
+title () {
+ txt="********** $1 **********"
+ printf "\n%*s%s\n\n" $((($COLUMNS-${#txt})/2)) "" "$txt"
+}
+
+bundle_install_trylocal() {
+ (
+ set -e
+ echo "(Running bundle install --local. 'could not find package' messages are OK.)"
+ if ! bundle install --local --no-deployment; then
+ echo "(Running bundle install again, without --local.)"
+ bundle install --no-deployment
+ fi
+ bundle package --all
+ )
+}
+
+install_doc() {
+ cd "$WORKSPACE/doc" \
+ && bundle_install_trylocal \
+ && rm -rf .site
+}
+do_install doc
+
+install_gem() {
+ gemname=$1
+ srcpath=$2
+ with_test_gemset gem_uninstall_if_exists "$gemname" \
+ && cd "$WORKSPACE/$srcpath" \
+ && bundle_install_trylocal \
+ && gem build "$gemname.gemspec" \
+ && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+}
+
+install_ruby_sdk() {
+ install_gem arvados sdk/ruby
+}
+do_install sdk/ruby ruby_sdk
+
+install_perl_sdk() {
+ cd "$WORKSPACE/sdk/perl" \
+ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
+ && make install INSTALLDIRS=perl
+}
+do_install sdk/perl perl_sdk
+
+install_cli() {
+ install_gem arvados-cli sdk/cli
+}
+do_install sdk/cli cli
+
+install_login-sync() {
+ install_gem arvados-login-sync services/login-sync
+}
+do_install services/login-sync login-sync
+
+# Install the Python SDK early. Various other test suites (like
+# keepproxy) bring up run_test_server.py, which imports the arvados
+# module. We can't actually *test* the Python SDK yet though, because
+# its own test suite brings up some of those other programs (like
+# keepproxy).
+declare -a pythonstuff
+pythonstuff=(
+ sdk/pam
+ sdk/python
+ sdk/cwl
+ services/fuse
+ services/nodemanager
+ tools/crunchstat-summary
+ )
+for p in "${pythonstuff[@]}"
+do
+ do_install "$p" pip
+done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
+
+install_apiserver() {
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle_install_trylocal
+
+ rm -f config/environments/test.rb
+ cp config/environments/test.rb.example config/environments/test.rb
+
+ if [ -n "$CONFIGSRC" ]
+ then
+ for f in database.yml application.yml
+ do
+ cp "$CONFIGSRC/$f" config/ || fatal "$f"
+ done
+ fi
+
+ # Fill in a random secret_token and blob_signing_key for testing
+ SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
+ BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
+
+ sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
+ sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
+
+ # Set up empty git repo (for git tests)
+ GITDIR=$(mktemp -d)
+ sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
+
+ rm -rf $GITDIR
+ mkdir -p $GITDIR/test
+ cd $GITDIR/test \
+ && git init \
+ && git config user.email "jenkins@ci.curoverse.com" \
+ && git config user.name "Jenkins, CI" \
+ && touch tmp \
+ && git add tmp \
+ && git commit -m 'initial commit'
+
+ # Clear out any lingering postgresql connections to the test
+ # database, so that we can drop it. This assumes the current user
+ # is a postgresql superuser.
+ cd "$WORKSPACE/services/api" \
+ && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle exec rake db:drop \
+ && RAILS_ENV=test bundle exec rake db:setup \
+ && RAILS_ENV=test bundle exec rake db:fixtures:load
+}
+do_install services/api apiserver
+
+declare -a gostuff
+gostuff=(
+ sdk/go/arvadosclient
+ sdk/go/blockdigest
+ sdk/go/manifest
+ sdk/go/streamer
+ sdk/go/crunchrunner
+ services/arv-git-httpd
+ services/crunchstat
+ services/keep-web
+ services/keepstore
+ sdk/go/keepclient
+ services/keepproxy
+ services/datamanager/summary
+ services/datamanager/collection
+ services/datamanager/keep
+ services/datamanager
+ services/crunch-dispatch-local
+ services/crunch-dispatch-slurm
+ services/crunch-run
+ tools/keep-rsync
+ )
+for g in "${gostuff[@]}"
+do
+ do_install "$g" go
+done
+
+install_workbench() {
+ cd "$WORKSPACE/apps/workbench" \
+ && mkdir -p tmp/cache \
+ && RAILS_ENV=test bundle_install_trylocal
+}
+do_install apps/workbench workbench
+
+test_doclinkchecker() {
+ (
+ set -e
+ cd "$WORKSPACE/doc"
+ ARVADOS_API_HOST=qr1hi.arvadosapi.com
+ # Make sure python-epydoc is installed or the next line won't
+ # do much good!
+ PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+ )
+}
+do_test doc doclinkchecker
+
+stop_services
+
+test_apiserver() {
+ rm -f "$WORKSPACE/services/api/git-commit.version"
+ cd "$WORKSPACE/services/api" \
+ && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]}
+}
+do_test services/api apiserver
+
+# Shortcut for when we're only running apiserver tests. This saves a bit of time,
+# because we don't need to start up the api server for subsequent tests.
+if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
+ rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
+ exit_cleanly
+fi
+
+start_api
+
+test_ruby_sdk() {
+ cd "$WORKSPACE/sdk/ruby" \
+ && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]}
+}
+do_test sdk/ruby ruby_sdk
+
+test_cli() {
+ cd "$WORKSPACE/sdk/cli" \
+ && mkdir -p /tmp/keep \
+ && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
+}
+do_test sdk/cli cli
+
+test_login-sync() {
+ cd "$WORKSPACE/services/login-sync" \
+ && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
+}
+do_test services/login-sync login-sync
+
+for p in "${pythonstuff[@]}"
+do
+ do_test "$p" pip
+done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
+
+for g in "${gostuff[@]}"
+do
+ do_test "$g" go
+done
+
+test_workbench() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
+}
+do_test apps/workbench workbench
+
+test_workbench_benchmark() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
+}
+do_test apps/workbench_benchmark workbench_benchmark
+
+test_workbench_profile() {
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
+ && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
+}
+do_test apps/workbench_profile workbench_profile
+
+exit_cleanly
--- /dev/null
+#!/bin/sh
+exec $TASK_KEEPMOUNT/$JOB_PARAMETER_CRUNCHRUNNER
installguide:
- Overview:
- install/index.html.textile.liquid
+ - Docker quick start:
+ - install/arvbox.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- install/install-sso.html.textile.liquid
"required": true,
"dataclass": "Collection"
},
- "sample_subdir": "$(dir $(samples))",
+ "sample_subdir": "$(dir $(sample))",
"read_pair": {
"value": {
"group": "sample_subdir",
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados-in-a-box
+...
+
+Arvbox is a Docker-based self-contained development, demonstration and testing environment for Arvados. It is not intended for production use.
+
+h2. Quick start
+
+<pre>
+$ git clone https://github.com/curoverse/arvados.git
+$ cd arvados/tools/arvbox/bin
+$ ./arvbox start localdemo
+</pre>
+
+h2. Requirements
+
+* Linux 3.x+ and Docker 1.9+
+* Minimum of 3 GiB of RAM + additional memory to run jobs
+* Minimum of 3 GiB of disk + storage for actual data
+
+h2. Usage
+
+<pre>
+$ arvbox
+Arvados-in-a-box http://arvados.org
+
+arvbox (build|start|run|open|shell|ip|stop|rebuild|reset|destroy|log|svrestart)
+
+build <config> build arvbox Docker image
+start|run <config> start arvbox container
+open open arvbox workbench in a web browser
+shell enter arvbox shell
+ip print arvbox docker container ip address
+host print arvbox published host
+status print some information about current arvbox
+stop stop arvbox container
+restart <config> stop, then run again
+rebuild <config> stop, build arvbox Docker image, run
+reset delete arvbox arvados data (be careful!)
+destroy delete all arvbox code and data (be careful!)
+log <service> tail log of specified service
+ls <options> list directories inside arvbox
+cat <files> get contents of files inside arvbox
+pipe run a bash script piped in from stdin
+sv <start|stop|restart> <service> change state of service inside arvbox
+clone <from> <to> clone an arvbox
+</pre>
+
+h2. Configs
+
+h3. dev
+
+Development configuration. Boots a complete Arvados environment inside the container. The "arvados", "arvado-dev" and "sso-devise-omniauth-provider" code directories along data directories "postgres", "var", "passenger" and "gems" are bind mounted from the host file system for easy access and persistence across container rebuilds. Services are bound to the Docker container's network IP address and can only be accessed on the local host.
+
+In "dev" mode, you can override the default autogenerated settings of Rails projects by adding "application.yml.override" to any Rails project (sso, api, workbench). This can be used to test out API server settings or point Workbench at an alternate API server.
+
+h3. localdemo
+
+Demo configuration. Boots a complete Arvados environment inside the container. Unlike the development configuration, code directories are included in the demo image, and data directories are stored in a separate data volume container. Services are bound to the Docker container's network IP address and can only be accessed on the local host.
+
+h3. test
+
+Run the test suite.
+
+h3. publicdev
+
+Publicly accessible development configuration. Similar to 'dev' except that service ports are published to the host's IP address and can accessed by anyone who can connect to the host system. See below for more information. WARNING! The public arvbox configuration is NOT SECURE and must not be placed on a public IP address or used for production work.
+
+h3. publicdemo
+
+Publicly accessible development configuration. Similar to 'localdemo' except that service ports are published to the host's IP address and can accessed by anyone who can connect to the host system. See below for more information. WARNING! The public arvbox configuration is NOT SECURE and must not be placed on a public IP address or used for production work.
+
+h2. Environment variables
+
+h3. ARVBOX_DOCKER
+
+The location of Dockerfile.base and associated files used by "arvbox build".
+default: result of $(readlink -f $(dirname $0)/../lib/arvbox/docker)
+
+h3. ARVBOX_CONTAINER
+
+The name of the Docker container to manipulate.
+default: arvbox
+
+h3. ARVBOX_BASE
+
+The base directory to store persistent data for arvbox containers.
+default: $HOME/.arvbox
+
+h3. ARVBOX_DATA
+
+The base directory to store persistent data for the current container.
+default: $ARVBOX_BASE/$ARVBOX_CONTAINER
+
+h3. ARVADOS_ROOT
+
+The root directory of the Arvados source tree
+default: $ARVBOX_DATA/arvados
+
+h3. ARVADOS_DEV_ROOT
+
+The root directory of the Arvados-dev source tree
+default: $ARVBOX_DATA/arvados-dev
+
+h3. SSO_ROOT
+
+The root directory of the SSO source tree
+default: $ARVBOX_DATA/sso-devise-omniauth-provider
+
+h3. ARVBOX_PUBLISH_IP
+
+The IP address on which to publish services when running in public configuration. Overrides default detection of the host's IP address.
+
+h2. Using Arvbox for Arvados development
+
+The "Arvbox section of Hacking Arvados":https://dev.arvados.org/projects/arvados/wiki/Arvbox has information about using Arvbox for Arvados development.
+
+h2. Making Arvbox accessible from other hosts
+
+In "dev" and "localdemo" mode, Arvbox can only be accessed on the same host it is running. To publish Arvbox service ports to the host's service ports and advertise the host's IP address for services, use @publicdev@ or @publicdemo@:
+
+<pre>
+$ arvbox rebuild publicdemo
+</pre>
+
+This attempts to auto-detect the correct IP address to use by taking the IP address of the default route device. If the auto-detection is wrong, you want to publish a hostname instead of a raw address, or you need to access it through a different device (such as a router or firewall), set @ARVBOX_PUBLISH_IP@ to the desire hostname or IP address.
+
+<pre>
+$ export ARVBOX_PUBLISH_IP=example.com
+$ arvbox rebuild publicdemo
+</pre>
+
+Note: this expects to bind the host's port 80 (http) for workbench, so you cannot have a conflicting web server already running on the host. It does not attempt to take bind the host's port 22 (ssh), as a result the arvbox ssh port is not published.
+
+h2. Notes
+
+Services are designed to install and auto-configure on start or restart. For example, the service script for keepstore always compiles keepstore from source and registers the daemon with the API server.
+
+Services are run with process supervision, so a service which exits will be restarted. Dependencies between services are handled by repeatedly trying and failing the service script until dependencies are fulfilled (by other service scripts) enabling the service script to complete.
Arvados components can be installed and configured in a number of different ways. Step-by-step instructions are available to perform a production installation from packages with manual configuration. This method assumes you have several (virtual) machines at your disposal for running the various Arvados components.
+* "Docker quick start":arvbox.html
* "Manual installation":install-manual-prerequisites.html
export HOME=$(pwd)
export RAILS_ENV=production
+## Uncomment and edit this line if your compute nodes have cgroup info
+## somewhere other than /sys/fs/cgroup (e.g., "/cgroup" for CentOS 6)
+#export CRUNCH_CGROUP_ROOT="/sys/fs/cgroup"
+
## Uncomment this line if your cluster uses self-signed SSL certificates:
#export ARVADOS_API_HOST_INSECURE=yes
my $job_api_token;
my $no_clear_tmp;
my $resume_stash;
+my $cgroup_root = "/sys/fs/cgroup";
my $docker_bin = "docker.io";
my $docker_run_args = "";
GetOptions('force-unlock' => \$force_unlock,
'job-api-token=s' => \$job_api_token,
'no-clear-tmp' => \$no_clear_tmp,
'resume-stash=s' => \$resume_stash,
+ 'cgroup-root=s' => \$cgroup_root,
'docker-bin=s' => \$docker_bin,
'docker-run-args=s' => \$docker_run_args,
);
$cmd = [$docker_bin, 'ps', '-q'];
}
Log(undef, "Sanity check is `@$cmd`");
-srun(["srun", "--nodes=\Q$ENV{SLURM_NNODES}\E", "--ntasks-per-node=1"],
- $cmd,
- {fork => 1});
-if ($? != 0) {
- Log(undef, "Sanity check failed: ".exit_status_s($?));
+my ($exited, $stdout, $stderr) = srun_sync(
+ ["srun", "--nodes=\Q$ENV{SLURM_NNODES}\E", "--ntasks-per-node=1"],
+ $cmd,
+ {label => "sanity check"});
+if ($exited != 0) {
+ Log(undef, "Sanity check failed: ".exit_status_s($exited));
exit EX_TEMPFAIL;
}
Log(undef, "Sanity check OK");
my $git_tar_count = 0;
if (!defined $no_clear_tmp) {
- # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
- Log (undef, "Clean work dirs");
-
- my $cleanpid = fork();
- if ($cleanpid == 0)
- {
- # Find FUSE mounts under $CRUNCH_TMP and unmount them.
- # Then clean up work directories.
- # TODO: When #5036 is done and widely deployed, we can limit mount's
- # -t option to simply fuse.keep.
- srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-ec', '-o', 'pipefail', 'mount -t fuse,fuse.keep | awk "(index(\$3, \"$CRUNCH_TMP\") == 1){print \$3}" | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid']);
- exit (1);
- }
- while (1)
- {
- last if $cleanpid == waitpid (-1, WNOHANG);
- freeze_if_want_freeze ($cleanpid);
- select (undef, undef, undef, 0.1);
- }
- if ($?) {
- Log(undef, "Clean work dirs: exit ".exit_status_s($?));
+ # Find FUSE mounts under $CRUNCH_TMP and unmount them. Then clean
+ # up work directories crunch_tmp/work, crunch_tmp/opt,
+ # crunch_tmp/src*.
+ #
+ # TODO: When #5036 is done and widely deployed, we can limit mount's
+ # -t option to simply fuse.keep.
+ my ($exited, $stdout, $stderr) = srun_sync(
+ ["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
+ ['bash', '-ec', '-o', 'pipefail', 'mount -t fuse,fuse.keep | awk "(index(\$3, \"$CRUNCH_TMP\") == 1){print \$3}" | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid'],
+ {label => "clean work dirs"});
+ if ($exited != 0) {
exit(EX_RETRY_UNLOCKED);
}
}
arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load
fi
};
- my $docker_pid = fork();
- if ($docker_pid == 0)
- {
- srun (["srun", "--nodelist=" . join(',', @node)],
- ["/bin/bash", "-o", "pipefail", "-ec", $docker_install_script]);
- exit ($?);
- }
- while (1)
- {
- last if $docker_pid == waitpid (-1, WNOHANG);
- freeze_if_want_freeze ($docker_pid);
- select (undef, undef, undef, 0.1);
- }
- if ($? != 0)
+
+ my ($exited, $stdout, $stderr) = srun_sync(
+ ["srun", "--nodelist=" . join(',', @node)],
+ ["/bin/bash", "-o", "pipefail", "-ec", $docker_install_script],
+ {label => "load docker image"});
+ if ($exited != 0)
{
- Log(undef, "Installing Docker image from $docker_locator exited " . exit_status_s($?));
exit(EX_RETRY_UNLOCKED);
}
# Determine whether this version of Docker supports memory+swap limits.
- srun(["srun", "--nodelist=" . $node[0]],
- ["/bin/sh", "-ec", "$docker_bin run --help | grep -qe --memory-swap="],
- {fork => 1});
- $docker_limitmem = ($? == 0);
+ ($exited, $stdout, $stderr) = srun_sync(
+ ["srun", "--nodes=1"],
+ [$docker_bin, 'run', '--help'],
+ {label => "check --memory-swap feature"});
+ $docker_limitmem = ($stdout =~ /--memory-swap/);
# Find a non-root Docker user to use.
# Tries the default user for the container, then 'crunch', then 'nobody',
# Docker containers.
my @tryusers = ("", "crunch", "nobody");
foreach my $try_user (@tryusers) {
+ my $label;
my $try_user_arg;
if ($try_user eq "") {
- Log(undef, "Checking if container default user is not UID 0");
+ $label = "check whether default user is UID 0";
$try_user_arg = "";
} else {
- Log(undef, "Checking if user '$try_user' is not UID 0");
+ $label = "check whether user '$try_user' is UID 0";
$try_user_arg = "--user=$try_user";
}
- srun(["srun", "--nodelist=" . $node[0]],
- ["/bin/sh", "-ec",
- "a=`$docker_bin run $docker_run_args $try_user_arg $docker_hash id --user` && " .
- " test \$a -ne 0"],
- {fork => 1});
- if ($? == 0) {
+ my ($exited, $stdout, $stderr) = srun_sync(
+ ["srun", "--nodes=1"],
+ ["/bin/sh", "-ec",
+ "$docker_bin run $docker_run_args $try_user_arg $docker_hash id --user"],
+ {label => $label});
+ chomp($stdout);
+ if ($exited == 0 && $stdout =~ /^\d+$/ && $stdout > 0) {
$dockeruserarg = $try_user_arg;
if ($try_user eq "") {
Log(undef, "Container will run with default user");
}
}
else {
- my $install_exited;
+ my $exited;
my $install_script_tries_left = 3;
for (my $attempts = 0; $attempts < 3; $attempts++) {
- Log(undef, "Run install script on all workers");
-
my @srunargs = ("srun",
"--nodelist=$nodelist",
"-D", $ENV{'TMPDIR'}, "--job-name=$job_id");
"mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
$ENV{"CRUNCH_GIT_ARCHIVE_HASH"} = md5_hex($git_archive);
- my ($install_stderr_r, $install_stderr_w);
- pipe $install_stderr_r, $install_stderr_w or croak("pipe() failed: $!");
- set_nonblocking($install_stderr_r);
- my $installpid = fork();
- if ($installpid == 0)
- {
- close($install_stderr_r);
- fcntl($install_stderr_w, F_SETFL, 0) or croak($!); # no close-on-exec
- open(STDOUT, ">&", $install_stderr_w);
- open(STDERR, ">&", $install_stderr_w);
- srun (\@srunargs, \@execargs, {}, $build_script . $git_archive);
- exit (1);
- }
- close($install_stderr_w);
- # Tell freeze_if_want_freeze how to kill the child, otherwise the
- # "waitpid(installpid)" loop won't get interrupted by a freeze:
- $proc{$installpid} = {};
- my $stderr_buf = '';
- # Track whether anything appears on stderr other than slurm errors
- # ("srun: ...") and the "starting: ..." message printed by the
- # srun subroutine itself:
+ my ($stdout, $stderr);
+ ($exited, $stdout, $stderr) = srun_sync(
+ \@srunargs, \@execargs,
+ {label => "run install script on all workers"},
+ $build_script . $git_archive);
+
my $stderr_anything_from_script = 0;
- my $match_our_own_errors = '^(srun: error: |starting: \[)';
- while ($installpid != waitpid(-1, WNOHANG)) {
- freeze_if_want_freeze ($installpid);
- # Wait up to 0.1 seconds for something to appear on stderr, then
- # do a non-blocking read.
- my $bits = fhbits($install_stderr_r);
- select ($bits, undef, $bits, 0.1);
- if (0 < sysread ($install_stderr_r, $stderr_buf, 8192, length($stderr_buf)))
- {
- while ($stderr_buf =~ /^(.*?)\n/) {
- my $line = $1;
- substr $stderr_buf, 0, 1+length($line), "";
- Log(undef, "stderr $line");
- if ($line !~ /$match_our_own_errors/) {
- $stderr_anything_from_script = 1;
- }
- }
- }
- }
- delete $proc{$installpid};
- $install_exited = $?;
- close($install_stderr_r);
- if (length($stderr_buf) > 0) {
- if ($stderr_buf !~ /$match_our_own_errors/) {
+ for my $line (split(/\n/, $stderr)) {
+ if ($line !~ /^(srun: error: |starting: \[)/) {
$stderr_anything_from_script = 1;
}
- Log(undef, "stderr $stderr_buf")
}
- Log (undef, "Install script exited ".exit_status_s($install_exited));
- last if $install_exited == 0 || $main::please_freeze;
+ last if $exited == 0 || $main::please_freeze;
+
# If the install script fails but doesn't print an error message,
# the next thing anyone is likely to do is just run it again in
# case it was a transient problem like "slurm communication fails
unlink($tar_filename);
}
- if ($install_exited != 0) {
+ if ($exited != 0) {
croak("Giving up");
}
}
@freeslot = (0..$#slot);
}
my $round_num_freeslots = scalar(@freeslot);
+print STDERR "crunch-job have ${round_num_freeslots} free slots for ${initial_tasks_this_level} initial tasks at this level, ".scalar(@node)." nodes, and ".scalar(@slot)." slots\n";
my %round_max_slots = ();
for (my $ii = $#freeslot; $ii >= 0; $ii--) {
{
my $containername = "$Jobstep->{arvados_task}->{uuid}-$Jobstep->{failures}";
my $cidfile = "$ENV{CRUNCH_TMP}/$containername.cid";
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
+ $command .= "crunchstat -cgroup-root=\Q$cgroup_root\E -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
$command .= "$docker_bin run $docker_run_args --name=$containername --attach=stdout --attach=stderr --attach=stdin -i \Q$dockeruserarg\E --cidfile=$cidfile --sig-proxy ";
# We only set memory limits if Docker lets us limit both memory and swap.
# Memory limits alone have been supported longer, but subprocesses tend
}
} else {
# Non-docker run
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
+ $command .= "crunchstat -cgroup-root=\Q$cgroup_root\E -poll=10000 ";
$command .= $stdbuf;
$command .= "perl - $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
}
next;
}
shift @freeslot;
- $proc{$childpid} = { jobstep => $id,
- time => time,
- slot => $childslot,
- jobstepname => "$job_id.$id.$childpid",
- };
+ $proc{$childpid} = {
+ jobstepidx => $id,
+ time => time,
+ slot => $childslot,
+ jobstepname => "$job_id.$id.$childpid",
+ };
croak ("assert failed: \$slot[$childslot]->{'pid'} exists") if exists $slot[$childslot]->{pid};
$slot[$childslot]->{pid} = $childpid;
sub reapchildren
{
- my $pid = waitpid (-1, WNOHANG);
- return 0 if $pid <= 0;
-
- my $whatslot = ($slot[$proc{$pid}->{slot}]->{node}->{name}
- . "."
- . $slot[$proc{$pid}->{slot}]->{cpu});
- my $jobstepid = $proc{$pid}->{jobstep};
- my $elapsed = time - $proc{$pid}->{time};
- my $Jobstep = $jobstep[$jobstepid];
-
- my $childstatus = $?;
- my $exitvalue = $childstatus >> 8;
- my $exitinfo = "exit ".exit_status_s($childstatus);
- $Jobstep->{'arvados_task'}->reload;
- my $task_success = $Jobstep->{'arvados_task'}->{success};
-
- Log ($jobstepid, "child $pid on $whatslot $exitinfo success=$task_success");
-
- if (!defined $task_success) {
- # task did not indicate one way or the other --> fail
- Log($jobstepid, sprintf(
- "ERROR: Task process exited %s, but never updated its task record to indicate success and record its output.",
- exit_status_s($childstatus)));
- $Jobstep->{'arvados_task'}->{success} = 0;
- $Jobstep->{'arvados_task'}->save;
- $task_success = 0;
- }
+ my $children_reaped = 0;
+ my @successful_task_uuids = ();
- if (!$task_success)
+ while((my $pid = waitpid (-1, WNOHANG)) > 0)
{
- my $temporary_fail;
- $temporary_fail ||= $Jobstep->{tempfail};
- $temporary_fail ||= ($exitvalue == TASK_TEMPFAIL);
-
- ++$thisround_failed;
- ++$thisround_failed_multiple if $Jobstep->{'failures'} >= 1;
-
- # Check for signs of a failed or misconfigured node
- if (++$slot[$proc{$pid}->{slot}]->{node}->{losing_streak} >=
- 2+$slot[$proc{$pid}->{slot}]->{node}->{ncpus}) {
- # Don't count this against jobstep failure thresholds if this
- # node is already suspected faulty and srun exited quickly
- if ($slot[$proc{$pid}->{slot}]->{node}->{hold_until} &&
- $elapsed < 5) {
- Log ($jobstepid, "blaming failure on suspect node " .
- $slot[$proc{$pid}->{slot}]->{node}->{name});
- $temporary_fail ||= 1;
- }
- ban_node_by_slot($proc{$pid}->{slot});
+ my $childstatus = $?;
+
+ my $whatslot = ($slot[$proc{$pid}->{slot}]->{node}->{name}
+ . "."
+ . $slot[$proc{$pid}->{slot}]->{cpu});
+ my $jobstepidx = $proc{$pid}->{jobstepidx};
+
+ if (!WIFEXITED($childstatus))
+ {
+ # child did not exit (may be temporarily stopped)
+ Log ($jobstepidx, "child $pid did not actually exit in reapchildren, ignoring for now.");
+ next;
}
- Log ($jobstepid, sprintf('failure (#%d, %s) after %d seconds',
- ++$Jobstep->{'failures'},
- $temporary_fail ? 'temporary' : 'permanent',
- $elapsed));
+ $children_reaped++;
+ my $elapsed = time - $proc{$pid}->{time};
+ my $Jobstep = $jobstep[$jobstepidx];
+
+ my $exitvalue = $childstatus >> 8;
+ my $exitinfo = "exit ".exit_status_s($childstatus);
+ $Jobstep->{'arvados_task'}->reload;
+ my $task_success = $Jobstep->{'arvados_task'}->{success};
+
+ Log ($jobstepidx, "child $pid on $whatslot $exitinfo success=$task_success");
+
+ if (!defined $task_success) {
+ # task did not indicate one way or the other --> fail
+ Log($jobstepidx, sprintf(
+ "ERROR: Task process exited %s, but never updated its task record to indicate success and record its output.",
+ exit_status_s($childstatus)));
+ $Jobstep->{'arvados_task'}->{success} = 0;
+ $Jobstep->{'arvados_task'}->save;
+ $task_success = 0;
+ }
- if (!$temporary_fail || $Jobstep->{'failures'} >= 3) {
- # Give up on this task, and the whole job
- $main::success = 0;
+ if (!$task_success)
+ {
+ my $temporary_fail;
+ $temporary_fail ||= $Jobstep->{tempfail};
+ $temporary_fail ||= ($exitvalue == TASK_TEMPFAIL);
+
+ ++$thisround_failed;
+ ++$thisround_failed_multiple if $Jobstep->{'failures'} >= 1;
+
+ # Check for signs of a failed or misconfigured node
+ if (++$slot[$proc{$pid}->{slot}]->{node}->{losing_streak} >=
+ 2+$slot[$proc{$pid}->{slot}]->{node}->{ncpus}) {
+ # Don't count this against jobstep failure thresholds if this
+ # node is already suspected faulty and srun exited quickly
+ if ($slot[$proc{$pid}->{slot}]->{node}->{hold_until} &&
+ $elapsed < 5) {
+ Log ($jobstepidx, "blaming failure on suspect node " .
+ $slot[$proc{$pid}->{slot}]->{node}->{name});
+ $temporary_fail ||= 1;
+ }
+ ban_node_by_slot($proc{$pid}->{slot});
+ }
+
+ Log ($jobstepidx, sprintf('failure (#%d, %s) after %d seconds',
+ ++$Jobstep->{'failures'},
+ $temporary_fail ? 'temporary' : 'permanent',
+ $elapsed));
+
+ if (!$temporary_fail || $Jobstep->{'failures'} >= 3) {
+ # Give up on this task, and the whole job
+ $main::success = 0;
+ }
+ # Put this task back on the todo queue
+ push @jobstep_todo, $jobstepidx;
+ $Job->{'tasks_summary'}->{'failed'}++;
}
- # Put this task back on the todo queue
- push @jobstep_todo, $jobstepid;
- $Job->{'tasks_summary'}->{'failed'}++;
+ else # task_success
+ {
+ push @successful_task_uuids, $Jobstep->{'arvados_task'}->{uuid};
+ ++$thisround_succeeded;
+ $slot[$proc{$pid}->{slot}]->{node}->{losing_streak} = 0;
+ $slot[$proc{$pid}->{slot}]->{node}->{hold_until} = 0;
+ $slot[$proc{$pid}->{slot}]->{node}->{fail_count} = 0;
+ push @jobstep_done, $jobstepidx;
+ Log ($jobstepidx, "success in $elapsed seconds");
+ }
+ $Jobstep->{exitcode} = $childstatus;
+ $Jobstep->{finishtime} = time;
+ $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
+ $Jobstep->{'arvados_task'}->save;
+ process_stderr_final ($jobstepidx);
+ Log ($jobstepidx, sprintf("task output (%d bytes): %s",
+ length($Jobstep->{'arvados_task'}->{output}),
+ $Jobstep->{'arvados_task'}->{output}));
+
+ close $reader{$jobstepidx};
+ delete $reader{$jobstepidx};
+ delete $slot[$proc{$pid}->{slot}]->{pid};
+ push @freeslot, $proc{$pid}->{slot};
+ delete $proc{$pid};
+
+ $progress_is_dirty = 1;
}
- else
+
+ if (scalar(@successful_task_uuids) > 0)
{
- ++$thisround_succeeded;
- $slot[$proc{$pid}->{slot}]->{node}->{losing_streak} = 0;
- $slot[$proc{$pid}->{slot}]->{node}->{hold_until} = 0;
- $slot[$proc{$pid}->{slot}]->{node}->{fail_count} = 0;
- push @jobstep_done, $jobstepid;
- Log ($jobstepid, "success in $elapsed seconds");
- }
- $Jobstep->{exitcode} = $childstatus;
- $Jobstep->{finishtime} = time;
- $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime});
- $Jobstep->{'arvados_task'}->save;
- process_stderr ($jobstepid, $task_success);
- Log ($jobstepid, sprintf("task output (%d bytes): %s",
- length($Jobstep->{'arvados_task'}->{output}),
- $Jobstep->{'arvados_task'}->{output}));
-
- close $reader{$jobstepid};
- delete $reader{$jobstepid};
- delete $slot[$proc{$pid}->{slot}]->{pid};
- push @freeslot, $proc{$pid}->{slot};
- delete $proc{$pid};
-
- if ($task_success) {
+ Log (undef, sprintf("%d tasks exited (%d succeeded), checking for new tasks from API server.", $children_reaped, scalar(@successful_task_uuids)));
# Load new tasks
my $newtask_list = [];
my $newtask_results;
do {
$newtask_results = api_call(
"job_tasks/list",
- 'where' => {
- 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
- },
+ 'filters' => [["created_by_job_task_uuid","in",\@successful_task_uuids]],
'order' => 'qsequence',
'offset' => scalar(@$newtask_list),
- );
+ );
push(@$newtask_list, @{$newtask_results->{items}});
} while (@{$newtask_results->{items}});
+ Log (undef, sprintf("Got %d new tasks from API server.", scalar(@$newtask_list)));
foreach my $arvados_task (@$newtask_list) {
my $jobstep = {
'level' => $arvados_task->{'sequence'},
}
}
- $progress_is_dirty = 1;
- 1;
+ return $children_reaped;
}
sub check_refresh_wanted
{
my @stat = stat $ENV{"CRUNCH_REFRESH_TRIGGER"};
- if (@stat && $stat[9] > $latest_refresh) {
+ if (@stat &&
+ $stat[9] > $latest_refresh &&
+ # ...and we have actually locked the job record...
+ $job_id eq $Job->{'uuid'}) {
$latest_refresh = scalar time;
my $Job2 = api_call("jobs/get", uuid => $jobspec);
for my $attr ('cancelled_at',
# squeue check interval (15s) this should make the squeue check an
# infrequent event.
my $silent_procs = 0;
- for my $procinfo (values %proc)
+ for my $js (map {$jobstep[$_->{jobstepidx}]} values %proc)
{
- my $jobstep = $jobstep[$procinfo->{jobstep}];
- if ($jobstep->{stderr_at} < $last_squeue_check)
+ if (!exists($js->{stderr_at}))
+ {
+ $js->{stderr_at} = 0;
+ }
+ if ($js->{stderr_at} < $last_squeue_check)
{
$silent_procs++;
}
# use killem() on procs whose killtime is reached
while (my ($pid, $procinfo) = each %proc)
{
- my $jobstep = $jobstep[$procinfo->{jobstep}];
+ my $js = $jobstep[$procinfo->{jobstepidx}];
if (exists $procinfo->{killtime}
&& $procinfo->{killtime} <= time
- && $jobstep->{stderr_at} < $last_squeue_check)
+ && $js->{stderr_at} < $last_squeue_check)
{
my $sincewhen = "";
- if ($jobstep->{stderr_at}) {
- $sincewhen = " in last " . (time - $jobstep->{stderr_at}) . "s";
+ if ($js->{stderr_at}) {
+ $sincewhen = " in last " . (time - $js->{stderr_at}) . "s";
}
- Log($procinfo->{jobstep}, "killing orphaned srun process $pid (task not in slurm queue, no stderr received$sincewhen)");
+ Log($procinfo->{jobstepidx}, "killing orphaned srun process $pid (task not in slurm queue, no stderr received$sincewhen)");
killem ($pid);
}
}
# error/delay has caused the task to die without notifying srun,
# and we'll kill srun ourselves.
$procinfo->{killtime} = time + 30;
- Log($procinfo->{jobstep}, "notice: task is not in slurm queue but srun process $pid has not exited");
+ Log($procinfo->{jobstepidx}, "notice: task is not in slurm queue but srun process $pid has not exited");
}
}
}
sub readfrompipes
{
my $gotsome = 0;
- foreach my $job (keys %reader)
+ my %fd_job;
+ my $sel = IO::Select->new();
+ foreach my $jobstepidx (keys %reader)
+ {
+ my $fd = $reader{$jobstepidx};
+ $sel->add($fd);
+ $fd_job{$fd} = $jobstepidx;
+
+ if (my $stdout_fd = $jobstep[$jobstepidx]->{stdout_r}) {
+ $sel->add($stdout_fd);
+ $fd_job{$stdout_fd} = $jobstepidx;
+ }
+ }
+ # select on all reader fds with 0.1s timeout
+ my @ready_fds = $sel->can_read(0.1);
+ foreach my $fd (@ready_fds)
{
my $buf;
- if (0 < sysread ($reader{$job}, $buf, 65536))
+ if (0 < sysread ($fd, $buf, 65536))
{
+ $gotsome = 1;
print STDERR $buf if $ENV{CRUNCH_DEBUG};
- $jobstep[$job]->{stderr_at} = time;
- $jobstep[$job]->{stderr} .= $buf;
+
+ my $jobstepidx = $fd_job{$fd};
+ if ($jobstep[$jobstepidx]->{stdout_r} == $fd) {
+ $jobstep[$jobstepidx]->{stdout_captured} .= $buf;
+ next;
+ }
+
+ $jobstep[$jobstepidx]->{stderr_at} = time;
+ $jobstep[$jobstepidx]->{stderr} .= $buf;
# Consume everything up to the last \n
- preprocess_stderr ($job);
+ preprocess_stderr ($jobstepidx);
- if (length ($jobstep[$job]->{stderr}) > 16384)
+ if (length ($jobstep[$jobstepidx]->{stderr}) > 16384)
{
# If we get a lot of stderr without a newline, chop off the
# front to avoid letting our buffer grow indefinitely.
- substr ($jobstep[$job]->{stderr},
- 0, length($jobstep[$job]->{stderr}) - 8192) = "";
+ substr ($jobstep[$jobstepidx]->{stderr},
+ 0, length($jobstep[$jobstepidx]->{stderr}) - 8192) = "";
}
- $gotsome = 1;
}
}
return $gotsome;
}
+# Consume all full lines of stderr for a jobstep. Everything after the
+# last newline will remain in $jobstep[$jobstepidx]->{stderr} after
+# returning.
sub preprocess_stderr
{
- my $job = shift;
+ my $jobstepidx = shift;
- while ($jobstep[$job]->{stderr} =~ /^(.*?)\n/) {
+ while ($jobstep[$jobstepidx]->{stderr} =~ /^(.*?)\n/) {
my $line = $1;
- substr $jobstep[$job]->{stderr}, 0, 1+length($line), "";
- Log ($job, "stderr $line");
+ substr $jobstep[$jobstepidx]->{stderr}, 0, 1+length($line), "";
+ Log ($jobstepidx, "stderr $line");
if ($line =~ /srun: error: (SLURM job $ENV{SLURM_JOB_ID} has expired|Unable to confirm allocation for job $ENV{SLURM_JOB_ID})/) {
# whoa.
$main::please_freeze = 1;
}
+ elsif (!exists $jobstep[$jobstepidx]->{slotindex}) {
+ # Skip the following tempfail checks if this srun proc isn't
+ # attached to a particular worker slot.
+ }
elsif ($line =~ /srun: error: (Node failure on|Aborting, .*\bio error\b)/) {
- my $job_slot_index = $jobstep[$job]->{slotindex};
+ my $job_slot_index = $jobstep[$jobstepidx]->{slotindex};
$slot[$job_slot_index]->{node}->{fail_count}++;
- $jobstep[$job]->{tempfail} = 1;
+ $jobstep[$jobstepidx]->{tempfail} = 1;
ban_node_by_slot($job_slot_index);
}
elsif ($line =~ /srun: error: (Unable to create job step|.*: Communication connection failure)/) {
- $jobstep[$job]->{tempfail} = 1;
- ban_node_by_slot($jobstep[$job]->{slotindex});
+ $jobstep[$jobstepidx]->{tempfail} = 1;
+ ban_node_by_slot($jobstep[$jobstepidx]->{slotindex});
}
- elsif ($line =~ /arvados\.errors\.Keep/) {
- $jobstep[$job]->{tempfail} = 1;
+ elsif ($line =~ /\bKeep(Read|Write|Request)Error:/) {
+ $jobstep[$jobstepidx]->{tempfail} = 1;
}
}
}
-sub process_stderr
+sub process_stderr_final
{
- my $job = shift;
- my $task_success = shift;
- preprocess_stderr ($job);
+ my $jobstepidx = shift;
+ preprocess_stderr ($jobstepidx);
map {
- Log ($job, "stderr $_");
- } split ("\n", $jobstep[$job]->{stderr});
+ Log ($jobstepidx, "stderr $_");
+ } split ("\n", $jobstep[$jobstepidx]->{stderr});
+ $jobstep[$jobstepidx]->{stderr} = '';
}
sub fetch_block
}
if (!exists $proc{$_}->{"sent_$sig"})
{
- Log ($proc{$_}->{jobstep}, "sending 2x signal $sig to pid $_");
+ Log ($proc{$_}->{jobstepidx}, "sending 2x signal $sig to pid $_");
kill $sig, $_;
select (undef, undef, undef, 0.1);
if ($sig == 2)
return $log_pipe_pid;
}
-sub Log # ($jobstep_id, $logmessage)
+sub Log # ($jobstepidx, $logmessage)
{
- if ($_[1] =~ /\n/) {
+ my ($jobstepidx, $logmessage) = @_;
+ if ($logmessage =~ /\n/) {
for my $line (split (/\n/, $_[1])) {
- Log ($_[0], $line);
+ Log ($jobstepidx, $line);
}
return;
}
my $fh = select STDERR; $|=1; select $fh;
- my $message = sprintf ("%s %d %s %s", $job_id, $$, @_);
+ my $task_qseq = '';
+ if (defined($jobstepidx) && exists($jobstep[$jobstepidx]->{arvados_task})) {
+ $task_qseq = $jobstepidx;
+ }
+ my $message = sprintf ("%s %d %s %s", $job_id, $$, $task_qseq, $logmessage);
$message =~ s{([^ -\176])}{"\\" . sprintf ("%03o", ord($1))}ge;
$message .= "\n";
my $datetime;
}
+sub srun_sync
+{
+ my $srunargs = shift;
+ my $execargs = shift;
+ my $opts = shift || {};
+ my $stdin = shift;
+
+ my $label = exists $opts->{label} ? $opts->{label} : "@$execargs";
+ Log (undef, "$label: start");
+
+ my ($stderr_r, $stderr_w);
+ pipe $stderr_r, $stderr_w or croak("pipe() failed: $!");
+
+ my ($stdout_r, $stdout_w);
+ pipe $stdout_r, $stdout_w or croak("pipe() failed: $!");
+
+ my $srunpid = fork();
+ if ($srunpid == 0)
+ {
+ close($stderr_r);
+ close($stdout_r);
+ fcntl($stderr_w, F_SETFL, 0) or croak($!); # no close-on-exec
+ fcntl($stdout_w, F_SETFL, 0) or croak($!);
+ open(STDERR, ">&", $stderr_w);
+ open(STDOUT, ">&", $stdout_w);
+ srun ($srunargs, $execargs, $opts, $stdin);
+ exit (1);
+ }
+ close($stderr_w);
+ close($stdout_w);
+
+ set_nonblocking($stderr_r);
+ set_nonblocking($stdout_r);
+
+ # Add entries to @jobstep and %proc so check_squeue() and
+ # freeze_if_want_freeze() can treat it like a job task process.
+ push @jobstep, {
+ stderr => '',
+ stderr_at => 0,
+ stderr_captured => '',
+ stdout_r => $stdout_r,
+ stdout_captured => '',
+ };
+ my $jobstepidx = $#jobstep;
+ $proc{$srunpid} = {
+ jobstepidx => $jobstepidx,
+ };
+ $reader{$jobstepidx} = $stderr_r;
+
+ while ($srunpid != waitpid ($srunpid, WNOHANG)) {
+ my $busy = readfrompipes();
+ if (!$busy || ($latest_refresh + 2 < scalar time)) {
+ check_refresh_wanted();
+ check_squeue();
+ }
+ if (!$busy) {
+ select(undef, undef, undef, 0.1);
+ }
+ killem(keys %proc) if $main::please_freeze;
+ }
+ my $exited = $?;
+
+ 1 while readfrompipes();
+ process_stderr_final ($jobstepidx);
+
+ Log (undef, "$label: exit ".exit_status_s($exited));
+
+ close($stdout_r);
+ close($stderr_r);
+ delete $proc{$srunpid};
+ delete $reader{$jobstepidx};
+
+ my $j = pop @jobstep;
+ return ($exited, $j->{stdout_captured}, $j->{stderr_captured});
+}
+
+
sub srun
{
my $srunargs = shift;
#!/bin/sh
echo >&2 Failing mount stub was called
-exit 1
+exit 44
tryjobrecord j, binstubs: ['clean_fail']
end
assert_match /Failing mount stub was called/, err
- assert_match /Clean work dirs: exit 1\n$/, err
+ assert_match /clean work dirs: exit 44\n$/, err
assert_equal SPECIAL_EXIT[:EX_RETRY_UNLOCKED], $?.exitstatus
end
import arvados.events
import arvados.commands.keepdocker
import arvados.commands.run
+import arvados.collection
+import arvados.util
import cwltool.draft2tool
import cwltool.workflow
import cwltool.main
import logging
import re
import os
+import sys
from cwltool.process import get_feature
+from arvados.api import OrderedJsonModel
logger = logging.getLogger('arvados.cwl-runner')
logger.setLevel(logging.INFO)
-def arv_docker_get_image(api_client, dockerRequirement, pull_image):
+crunchrunner_pdh = "83db29f08544e1c319572a6bd971088a+140"
+crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
+certs_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/ca-certificates.crt"
+
+tmpdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.tmpdir\)=(.*)")
+outdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.outdir\)=(.*)")
+keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.keep\)=(.*)")
+
+
+def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
if not images:
imageId = cwltool.docker.get_image(dockerRequirement, pull_image)
- args = [image_name]
+ args = ["--project-uuid="+project_uuid, image_name]
if image_tag:
args.append(image_tag)
- logger.info("Uploading Docker image %s", ":".join(args))
+ logger.info("Uploading Docker image %s", ":".join(args[1:]))
arvados.commands.keepdocker.main(args)
return dockerRequirement["dockerImageId"]
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if docker_req and kwargs.get("use_container") is not False:
- runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image)
+ runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
+
+ resources = self.builder.resources
+ if resources is not None:
+ runtime_constraints["min_cores_per_node"] = resources.get("cores", 1)
+ runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
+ runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
try:
response = self.arvrunner.api.jobs().create(body={
+ "owner_uuid": self.arvrunner.project_uuid,
"script": "crunchrunner",
- "repository": kwargs["repository"],
+ "repository": "arvados",
"script_version": "master",
- "script_parameters": {"tasks": [script_parameters]},
+ "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6",
+ "script_parameters": {"tasks": [script_parameters], "crunchrunner": crunchrunner_pdh+"/crunchrunner"},
"runtime_constraints": runtime_constraints
}, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
try:
outputs = {}
if record["output"]:
+ logc = arvados.collection.Collection(record["log"])
+ log = logc.open(logc.keys()[0])
+ tmpdir = None
+ outdir = None
+ keepdir = None
+ for l in log.readlines():
+ g = tmpdirre.match(l)
+ if g:
+ tmpdir = g.group(1)
+ g = outdirre.match(l)
+ if g:
+ outdir = g.group(1)
+ g = keepre.match(l)
+ if g:
+ keepdir = g.group(1)
+ if tmpdir and outdir and keepdir:
+ break
+
+ self.builder.outdir = outdir
+ self.builder.pathmapper.keepdir = keepdir
outputs = self.collect_outputs("keep:" + record["output"])
except Exception as e:
logger.exception("Got exception while collecting job outputs:")
arvrunner.api,
dry_run=kwargs.get("dry_run"),
num_retries=3,
- fnPattern="$(task.keep)/%s/%s")
+ fnPattern="$(task.keep)/%s/%s",
+ project=arvrunner.project_uuid)
for src, ab, st in uploadfiles:
arvrunner.add_uploaded(src, (ab, st.fn))
self._pathmap[src] = (ab, st.fn)
+ self.keepdir = None
+
+ def reversemap(self, target):
+ if target.startswith("keep:"):
+ return (target, target)
+ elif self.keepdir and target.startswith(self.keepdir):
+ return (target, "keep:" + target[len(self.keepdir)+1:])
+ else:
+ return super(ArvPathMapper, self).reversemap(target)
class ArvadosCommandTool(cwltool.draft2tool.CommandLineTool):
def __init__(self, arvrunner, toolpath_object, **kwargs):
- super(ArvadosCommandTool, self).__init__(toolpath_object, outdir="$(task.outdir)", tmpdir="$(task.tmpdir)", **kwargs)
+ super(ArvadosCommandTool, self).__init__(toolpath_object, **kwargs)
self.arvrunner = arvrunner
def makeJobRunner(self):
def on_message(self, event):
if "object_uuid" in event:
- if event["object_uuid"] in self.jobs and event["event_type"] == "update":
- if event["properties"]["new_attributes"]["state"] == "Running" and self.jobs[event["object_uuid"]].running is False:
- uuid = event["object_uuid"]
- with self.lock:
- j = self.jobs[uuid]
- logger.info("Job %s (%s) is Running", j.name, uuid)
- j.running = True
- j.update_pipeline_component(event["properties"]["new_attributes"])
- elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"):
- uuid = event["object_uuid"]
- try:
- self.cond.acquire()
- j = self.jobs[uuid]
- logger.info("Job %s (%s) is %s", j.name, uuid, event["properties"]["new_attributes"]["state"])
- j.done(event["properties"]["new_attributes"])
- self.cond.notify()
- finally:
- self.cond.release()
+ if event["object_uuid"] in self.jobs and event["event_type"] == "update":
+ if event["properties"]["new_attributes"]["state"] == "Running" and self.jobs[event["object_uuid"]].running is False:
+ uuid = event["object_uuid"]
+ with self.lock:
+ j = self.jobs[uuid]
+ logger.info("Job %s (%s) is Running", j.name, uuid)
+ j.running = True
+ j.update_pipeline_component(event["properties"]["new_attributes"])
+ elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"):
+ uuid = event["object_uuid"]
+ try:
+ self.cond.acquire()
+ j = self.jobs[uuid]
+ logger.info("Job %s (%s) is %s", j.name, uuid, event["properties"]["new_attributes"]["state"])
+ j.done(event["properties"]["new_attributes"])
+ self.cond.notify()
+ finally:
+ self.cond.release()
def get_uploaded(self):
return self.uploaded.copy()
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
- self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
- "components": {},
- "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
+ try:
+ self.api.collections().get(uuid=crunchrunner_pdh).execute()
+ except arvados.errors.ApiError as e:
+ import httplib2
+ h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
+ resp, content = h.request(crunchrunner_download, "GET")
+ resp2, content2 = h.request(certs_download, "GET")
+ with arvados.collection.Collection() as col:
+ with col.open("crunchrunner", "w") as f:
+ f.write(content)
+ with col.open("ca-certificates.crt", "w") as f:
+ f.write(content2)
+
+ col.save_new("crunchrunner binary", ensure_unique_name=True)
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
- kwargs["repository"] = args.repository
+
+ kwargs["outdir"] = "$(task.outdir)"
+ kwargs["tmpdir"] = "$(task.tmpdir)"
+
+ useruuid = self.api.users().current().execute()["uuid"]
+ self.project_uuid = args.project_uuid if args.project_uuid else useruuid
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
else:
+ self.pipeline = self.api.pipeline_instances().create(
+ body={
+ "owner_uuid": self.project_uuid,
+ "name": shortname(tool.tool["id"]),
+ "components": {},
+ "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
+
+ logger.info("Pipeline instance %s", self.pipeline["uuid"])
+
jobiter = tool.job(job_order,
- input_basedir,
- self.output_callback,
- **kwargs)
+ input_basedir,
+ self.output_callback,
+ docker_outdir="$(task.outdir)",
+ **kwargs)
- for runnable in jobiter:
- if runnable:
- with self.lock:
+ try:
+ self.cond.acquire()
+ # Will continue to hold the lock for the duration of this code
+ # except when in cond.wait(), at which point on_message can update
+ # job state and process output callbacks.
+
+ for runnable in jobiter:
+ if runnable:
runnable.run(**kwargs)
- else:
- if self.jobs:
- try:
- self.cond.acquire()
- self.cond.wait()
- finally:
- self.cond.release()
else:
- logger.error("Workflow cannot make any more progress.")
- break
+ if self.jobs:
+ self.cond.wait(1)
+ else:
+ logger.error("Workflow is deadlocked, no runnable jobs and not waiting on any pending jobs.")
+ break
- while self.jobs:
- try:
- self.cond.acquire()
- self.cond.wait()
- finally:
- self.cond.release()
+ while self.jobs:
+ self.cond.wait(1)
- events.close()
+ events.close()
- if self.final_output is None:
- raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
+ if self.final_output is None:
+ raise cwltool.workflow.WorkflowException("Workflow did not return a result.")
+
+ # create final output collection
+ except:
+ if sys.exc_info()[0] is KeyboardInterrupt:
+ logger.error("Interrupted, marking pipeline as failed")
+ else:
+ logger.exception("Caught unhandled exception, marking pipeline as failed")
+ self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
+ body={"state": "Failed"}).execute(num_retries=self.num_retries)
+ finally:
+ self.cond.release()
return self.final_output
def main(args, stdout, stderr, api_client=None):
- runner = ArvCwlRunner(api_client=arvados.api('v1'))
args.insert(0, "--leave-outputs")
parser = cwltool.main.arg_parser()
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--enable-reuse", action="store_true",
- default=False, dest="enable_reuse",
+ default=True, dest="enable_reuse",
help="")
exgroup.add_argument("--disable-reuse", action="store_false",
- default=False, dest="enable_reuse",
+ default=True, dest="enable_reuse",
help="")
+ parser.add_argument("--project-uuid", type=str, help="Project that will own the workflow jobs")
- parser.add_argument('--repository', type=str, default="peter/crunchrunner", help="Repository containing the 'crunchrunner' program.")
+ try:
+ runner = ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
+ except Exception as e:
+ logger.error(e)
+ return 1
return cwltool.main.main(args, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser)
'bin/arvados-cwl-runner'
],
install_requires=[
- 'cwltool>=1.0.20160129152024',
- 'arvados-python-client>=0.1.20160122132348'
+ 'cwltool>=1.0.20160311170456',
+ 'arvados-python-client>=0.1.20160219154918'
],
+ test_suite='tests',
+ tests_require=['mock>=1.0'],
zip_safe=True,
cmdclass={'egg_info': tagger},
)
--- /dev/null
+#!/bin/sh
+
+if ! which arvbox >/dev/null ; then
+ export PATH=$PATH:$(readlink -f $(dirname $0)/../../tools/arvbox/bin)
+fi
+
+reset_container=1
+leave_running=0
+config=dev
+
+while test -n "$1" ; do
+ arg="$1"
+ case "$arg" in
+ --no-reset-container)
+ reset_container=0
+ shift
+ ;;
+ --leave-running)
+ leave_running=1
+ shift
+ ;;
+ --config)
+ config=$2
+ shift ; shift
+ ;;
+ -*)
+ break
+ ;;
+ esac
+done
+
+if test -z "$ARVBOX_CONTAINER" ; then
+ export ARVBOX_CONTAINER=cwltest
+fi
+
+if test $reset_container = 1 ; then
+ arvbox reset -f
+fi
+
+arvbox start $config
+
+arvbox pipe <<EOF
+set -eu -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/arvados/sdk/cwl
+python setup.py sdist
+pip_install \$(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
+
+mkdir -p /tmp/cwltest
+cd /tmp/cwltest
+if ! test -d common-workflow-language ; then
+ git clone https://github.com/common-workflow-language/common-workflow-language.git
+fi
+cd common-workflow-language
+git pull
+export ARVADOS_API_HOST=localhost:8000
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
+env
+exec ./run_test.sh "$@"
+EOF
+
+CODE=$?
+
+if test $leave_running = 0 ; then
+ arvbox stop
+fi
+
+exit $CODE
--- /dev/null
+import unittest
+import mock
+import arvados_cwl
+
+class TestJob(unittest.TestCase):
+
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ def test_run(self):
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ tool = {
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls"
+ }
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
+ arvtool.formatgraph = None
+ for j in arvtool.job({}, "", mock.MagicMock()):
+ j.run()
+ runner.api.jobs().create.assert_called_with(body={
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'runtime_constraints': {},
+ 'script_parameters': {
+ 'tasks': [{
+ 'task.env': {'TMPDIR': '$(task.tmpdir)'},
+ 'command': ['ls']
+ }],
+ 'crunchrunner': '83db29f08544e1c319572a6bd971088a+140/crunchrunner'
+ },
+ 'script_version': 'master',
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script': 'crunchrunner',
+ 'runtime_constraints': {
+ 'min_cores_per_node': 1,
+ 'min_ram_mb_per_node': 1024,
+ 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
+ }
+ }, find_or_create=True)
+
+ # The test passes some fields in builder.resources
+ # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ def test_resource_requirements(self):
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ tool = {
+ "inputs": [],
+ "outputs": [],
+ "hints": [{
+ "class": "ResourceRequirement",
+ "coresMin": 3,
+ "ramMin": 3000,
+ "tmpdirMin": 4000
+ }],
+ "baseCommand": "ls"
+ }
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
+ arvtool.formatgraph = None
+ for j in arvtool.job({}, "", mock.MagicMock()):
+ j.run()
+ runner.api.jobs().create.assert_called_with(body={
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'runtime_constraints': {},
+ 'script_parameters': {
+ 'tasks': [{
+ 'task.env': {'TMPDIR': '$(task.tmpdir)'},
+ 'command': ['ls']
+ }],
+ 'crunchrunner': '83db29f08544e1c319572a6bd971088a+140/crunchrunner'
+ },
+ 'script_version': 'master',
+ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
+ 'repository': 'arvados',
+ 'script': 'crunchrunner',
+ 'runtime_constraints': {
+ 'min_cores_per_node': 3,
+ 'min_ram_mb_per_node': 3000,
+ 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
+ }
+ }, find_or_create=True)
package main
import (
+ "crypto/x509"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "io/ioutil"
"log"
+ "net/http"
"os"
"os/exec"
"os/signal"
+ "path"
"strings"
"syscall"
)
"$(task.outdir)": outdir,
"$(task.keep)": keepmount}
+ log.Printf("crunchrunner: $(task.tmpdir)=%v", tmpdir)
+ log.Printf("crunchrunner: $(task.outdir)=%v", outdir)
+ log.Printf("crunchrunner: $(task.keep)=%v", keepmount)
+
// Set up subprocess
for k, v := range taskp.Command {
taskp.Command[k] = substitute(v, replacements)
log.Fatal(err)
}
+ certpath := path.Join(path.Dir(os.Args[0]), "ca-certificates.crt")
+ certdata, err := ioutil.ReadFile(certpath)
+ if err == nil {
+ log.Printf("Using TLS certificates at %v", certpath)
+ certs := x509.NewCertPool()
+ certs.AppendCertsFromPEM(certdata)
+ api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
+ }
+
jobUuid := os.Getenv("JOB_UUID")
taskUuid := os.Getenv("TASK_UUID")
tmpdir := os.Getenv("TASK_WORK")
svc = apiclient_discovery.build('arvados', version, **kwargs)
svc.api_token = token
+ svc.insecure = insecure
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
return svc
import sys
import logging
import tempfile
+import urlparse
import arvados
import arvados.config
copy_opts.add_argument(
'--project-uuid', dest='project_uuid',
help='The UUID of the project at the destination to which the pipeline should be copied.')
+ copy_opts.add_argument(
+ '--allow-git-http-src', action="store_true",
+ help='Allow cloning git repositories over insecure http')
+ copy_opts.add_argument(
+ '--allow-git-http-dst', action="store_true",
+ help='Allow pushing git repositories over insecure http')
+
copy_opts.add_argument(
'object_uuid',
help='The UUID of the object to be copied.')
c['manifest_text'] = dst_manifest
return create_collection_from(c, src, dst, args)
+def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_http_opt):
+ r = api.repositories().list(
+ filters=[['name', '=', repo_name]]).execute(num_retries=retries)
+ if r['items_available'] != 1:
+ raise Exception('cannot identify repo {}; {} repos found'
+ .format(repo_name, r['items_available']))
+
+ https_url = [c for c in r['items'][0]["clone_urls"] if c.startswith("https:")]
+ http_url = [c for c in r['items'][0]["clone_urls"] if c.startswith("http:")]
+ other_url = [c for c in r['items'][0]["clone_urls"] if not c.startswith("http")]
+
+ priority = https_url + other_url + http_url
+
+ git_config = []
+ git_url = None
+ for url in priority:
+ if url.startswith("http"):
+ u = urlparse.urlsplit(url)
+ baseurl = urlparse.urlunsplit((u.scheme, u.netloc, "", "", ""))
+ git_config = ["-c", "credential.%s/.username=none" % baseurl,
+ "-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
+ else:
+ git_config = []
+
+ try:
+ logger.debug("trying %s", url)
+ arvados.util.run_command(["git"] + git_config + ["ls-remote", url],
+ env={"HOME": os.environ["HOME"],
+ "ARVADOS_API_TOKEN": api.api_token,
+ "GIT_ASKPASS": "/bin/false"})
+ except arvados.errors.CommandFailedError:
+ pass
+ else:
+ git_url = url
+ break
+
+ if not git_url:
+ raise Exception('Cannot access git repository, tried {}'
+ .format(priority))
+
+ if git_url.startswith("http:"):
+ if allow_insecure_http:
+ logger.warn("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
+ else:
+ raise Exception("Refusing to use insecure git url %s, use %s if you really want this." % (git_url, allow_insecure_http_opt))
+
+ return (git_url, git_config)
+
+
# copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version, args)
#
# Copies commits from git repository 'src_git_repo' on Arvados
#
def copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version, args):
# Identify the fetch and push URLs for the git repositories.
- r = src.repositories().list(
- filters=[['name', '=', src_git_repo]]).execute(num_retries=args.retries)
- if r['items_available'] != 1:
- raise Exception('cannot identify source repo {}; {} repos found'
- .format(src_git_repo, r['items_available']))
- src_git_url = r['items'][0]['fetch_url']
- logger.debug('src_git_url: {}'.format(src_git_url))
- r = dst.repositories().list(
- filters=[['name', '=', dst_git_repo]]).execute(num_retries=args.retries)
- if r['items_available'] != 1:
- raise Exception('cannot identify destination repo {}; {} repos found'
- .format(dst_git_repo, r['items_available']))
- dst_git_push_url = r['items'][0]['push_url']
- logger.debug('dst_git_push_url: {}'.format(dst_git_push_url))
+ (src_git_url, src_git_config) = select_git_url(src, src_git_repo, args.retries, args.allow_git_http_src, "--allow-git-http-src")
+ (dst_git_url, dst_git_config) = select_git_url(dst, dst_git_repo, args.retries, args.allow_git_http_dst, "--allow-git-http-dst")
+
+ logger.debug('src_git_url: {}'.format(src_git_url))
+ logger.debug('dst_git_url: {}'.format(dst_git_url))
dst_branch = re.sub(r'\W+', '_', "{}_{}".format(src_git_url, script_version))
if src_git_repo not in local_repo_dir:
local_repo_dir[src_git_repo] = tempfile.mkdtemp()
arvados.util.run_command(
- ["git", "clone", "--bare", src_git_url,
+ ["git"] + src_git_config + ["clone", "--bare", src_git_url,
local_repo_dir[src_git_repo]],
- cwd=os.path.dirname(local_repo_dir[src_git_repo]))
+ cwd=os.path.dirname(local_repo_dir[src_git_repo]),
+ env={"HOME": os.environ["HOME"],
+ "ARVADOS_API_TOKEN": src.api_token,
+ "GIT_ASKPASS": "/bin/false"})
arvados.util.run_command(
- ["git", "remote", "add", "dst", dst_git_push_url],
+ ["git", "remote", "add", "dst", dst_git_url],
cwd=local_repo_dir[src_git_repo])
arvados.util.run_command(
["git", "branch", dst_branch, script_version],
cwd=local_repo_dir[src_git_repo])
- arvados.util.run_command(["git", "push", "dst", dst_branch],
- cwd=local_repo_dir[src_git_repo])
+ arvados.util.run_command(["git"] + dst_git_config + ["push", "dst", dst_branch],
+ cwd=local_repo_dir[src_git_repo],
+ env={"HOME": os.environ["HOME"],
+ "ARVADOS_API_TOKEN": dst.api_token,
+ "GIT_ASKPASS": "/bin/false"})
def copy_docker_images(pipeline, src, dst, args):
"""Copy any docker images named in the pipeline components'
"lib/arvados/collection.rb", "lib/arvados/keep.rb",
"README", "LICENSE-2.0.txt"]
s.required_ruby_version = '>= 2.1.0'
+ # activesupport <4.2.6 only because https://dev.arvados.org/issues/8222
+ s.add_dependency('activesupport', '>= 3.2.13', '< 4.2.6')
+ s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
s.add_dependency('google-api-client', '~> 0.6.3', '>= 0.6.3')
- s.add_dependency('activesupport', '>= 3.2.13')
s.add_dependency('json', '~> 1.7', '>= 1.7.7')
- s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
s.add_runtime_dependency('jwt', '>= 0.1.5', '< 1.0.0')
s.homepage =
'https://arvados.org'
def find_objects_for_index
# Here we are deliberately less helpful about searching for client
# authorizations. We look up tokens belonging to the current user
- # and filter by exact matches on api_token and scopes.
+ # and filter by exact matches on uuid, api_token, and scopes.
wanted_scopes = []
if @filters
wanted_scopes.concat(@filters.map { |attr, operator, operand|
((attr == 'scopes') and (operator == '=')) ? operand : nil
})
@filters.select! { |attr, operator, operand|
- ((attr == 'uuid') and (operator == '=')) || ((attr == 'api_token') and (operator == '='))
+ operator == '=' && (attr == 'uuid' || attr == 'api_token')
}
end
if @where
wanted_scopes << @where['scopes']
- @where.select! { |attr, val| attr == 'uuid' }
+ @where.select! { |attr, val|
+ # "where":{"uuid":"zzzzz-zzzzz-zzzzzzzzzzzzzzz"} is OK but
+ # "where":{"api_client_id":1} is not supported
+ # "where":{"uuid":["contains","-"]} is not supported
+ # "where":{"uuid":["uuid1","uuid2","uuid3"]} is not supported
+ val.is_a?(String) && (attr == 'uuid' || attr == 'api_token')
+ }
end
@objects = model_class.
includes(:user, :api_client).
end
def find_object_by_uuid
- @object = model_class.where(uuid: (params[:uuid] || params[:id])).first
+ uuid_param = params[:uuid] || params[:id]
+ if (uuid_param != current_api_client_authorization.andand.uuid and
+ not Thread.current[:api_client].andand.is_trusted)
+ return forbidden
+ end
+ @limit = 1
+ @offset = 0
+ @orders = []
+ @where = {}
+ @filters = [['uuid', '=', uuid_param]]
+ find_objects_for_index
+ @object = @objects.first
end
def current_api_client_is_trusted
- unless Thread.current[:api_client].andand.is_trusted
- if params["action"] == "show"
- if @object and @object['api_token'] == current_api_client_authorization.andand.api_token
- return true
- end
- elsif params["action"] == "index" and @objects.andand.size == 1
- filters = @filters.map{|f|f.first}.uniq
- if ['uuid'] == filters
- return true if @objects.first['api_token'] == current_api_client_authorization.andand.api_token
- elsif ['api_token'] == filters
- return true if @objects.first[:user_id] = current_user.id
- end
- end
- send_error('Forbidden: this API client cannot manipulate other clients\' access tokens.',
- status: 403)
+ if Thread.current[:api_client].andand.is_trusted
+ return true
+ end
+ # A non-trusted client can do a search for its own token if it
+ # explicitly restricts the search to its own UUID or api_token.
+ # Any other kind of query must return 403, even if it matches only
+ # the current token, because that's currently how Workbench knows
+ # (after searching on scopes) the difference between "the token
+ # I'm using now *is* the only sharing token for this collection"
+ # (403) and "my token is trusted, and there is one sharing token
+ # for this collection" (200).
+ #
+ # The @filters test here also prevents a non-trusted token from
+ # filtering on its own scopes, and discovering whether any _other_
+ # equally scoped tokens exist (403=yes, 200=no).
+ if (@objects.andand.count == 1 and
+ @objects.first.uuid == current_api_client_authorization.andand.uuid and
+ (@filters.map(&:first) & %w(uuid api_token)).any?)
+ return true
end
+ forbidden
+ end
+
+ def forbidden
+ send_error('Forbidden: this API client cannot manipulate other clients\' access tokens.',
+ status: 403)
end
end
@repo_info.values.each do |repo|
repo[:user_permissions].each do |user_uuid, user_perms|
if user_perms['can_manage']
- user_perms['gitolite_permissions'] = 'RW'
+ user_perms['gitolite_permissions'] = 'RW+'
user_perms['can_write'] = true
user_perms['can_read'] = true
elsif user_perms['can_write']
- user_perms['gitolite_permissions'] = 'RW'
+ user_perms['gitolite_permissions'] = 'RW+'
user_perms['can_read'] = true
elsif user_perms['can_read']
user_perms['gitolite_permissions'] = 'R'
def permission_to_update
(permission_to_create and
- not self.user_id_changed? and
- not self.owner_uuid_changed?)
+ not uuid_changed? and
+ not user_id_changed? and
+ not owner_uuid_changed?)
end
def log_update
# "git log".
source_version: false
+ crunch_log_partial_line_throttle_period: 5
development:
force_ssl: false
@docker_bin = ENV['CRUNCH_JOB_DOCKER_BIN']
@docker_run_args = ENV['CRUNCH_JOB_DOCKER_RUN_ARGS']
+ @cgroup_root = ENV['CRUNCH_CGROUP_ROOT']
@arvados_internal = Rails.configuration.git_internal_dir
if not File.exists? @arvados_internal
'--job', job.uuid,
'--git-dir', @arvados_internal]
+ if @cgroup_root
+ cmd_args += ['--cgroup-root', @cgroup_root]
+ end
+
if @docker_bin
cmd_args += ['--docker-bin', @docker_bin]
end
log_throttle_bytes_so_far: 0,
log_throttle_lines_so_far: 0,
log_throttle_bytes_skipped: 0,
+ log_throttle_partial_line_last_at: Time.new(0),
+ log_throttle_first_partial_line: true,
}
i.close
@todo_job_retries.delete(job.uuid)
message = false
linesize = line.size
if running_job[:log_throttle_is_open]
- running_job[:log_throttle_lines_so_far] += 1
- running_job[:log_throttle_bytes_so_far] += linesize
- running_job[:bytes_logged] += linesize
+ partial_line = false
+ skip_counts = false
+ matches = line.match(/^\S+ \S+ \d+ \d+ stderr (.*)/)
+ if matches and matches[1] and matches[1].start_with?('[...]') and matches[1].end_with?('[...]')
+ partial_line = true
+ if Time.now > running_job[:log_throttle_partial_line_last_at] + Rails.configuration.crunch_log_partial_line_throttle_period
+ running_job[:log_throttle_partial_line_last_at] = Time.now
+ else
+ skip_counts = true
+ end
+ end
+
+ if !skip_counts
+ running_job[:log_throttle_lines_so_far] += 1
+ running_job[:log_throttle_bytes_so_far] += linesize
+ running_job[:bytes_logged] += linesize
+ end
if (running_job[:bytes_logged] >
Rails.configuration.crunch_limit_log_bytes_per_job)
elsif (running_job[:log_throttle_bytes_so_far] >
Rails.configuration.crunch_log_throttle_bytes)
remaining_time = running_job[:log_throttle_reset_time] - Time.now
- message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_bytes). Logging will be silenced for the next #{remaining_time.round} seconds.\n"
+ message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_bytes). Logging will be silenced for the next #{remaining_time.round} seconds."
running_job[:log_throttle_is_open] = false
elsif (running_job[:log_throttle_lines_so_far] >
Rails.configuration.crunch_log_throttle_lines)
remaining_time = running_job[:log_throttle_reset_time] - Time.now
- message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_lines} lines per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_lines), logging will be silenced for the next #{remaining_time.round} seconds.\n"
+ message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_lines} lines per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_lines), logging will be silenced for the next #{remaining_time.round} seconds."
running_job[:log_throttle_is_open] = false
+
+ elsif partial_line and running_job[:log_throttle_first_partial_line]
+ running_job[:log_throttle_first_partial_line] = false
+ message = "Rate-limiting partial segments of long lines to one every #{Rails.configuration.crunch_log_partial_line_throttle_period} seconds."
end
end
if message
# Yes, write to logs, but use our "rate exceeded" message
# instead of the log message that exceeded the limit.
+ message += " A complete log is still being written to Keep, and will be available when the job finishes.\n"
line.replace message
true
+ elsif partial_line
+ false
else
running_job[:log_throttle_is_open]
end
j[:log_throttle_lines_so_far] = 0
j[:log_throttle_bytes_skipped] = 0
j[:log_throttle_is_open] = true
+ j[:log_throttle_partial_line_last_at] = Time.new(0)
+ j[:log_throttle_first_partial_line] = true
end
j[:buf].each do |stream, streambuf|
end
end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 403], # this belongs to the user of current session, but we can't get it by uuid
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} get #{token} token and expect #{status}" do
+ [# anyone can look up the token they're currently using
+ [:admin, :admin, 200, 200, 1],
+ [:active, :active, 200, 200, 1],
+ # cannot look up other tokens (even for same user) if not trustedclient
+ [:admin, :active, 403, 403],
+ [:admin, :admin_vm, 403, 403],
+ [:active, :admin, 403, 403],
+ # cannot look up other tokens for other users, regardless of trustedclient
+ [:admin_trustedclient, :active, 404, 200, 0],
+ [:active_trustedclient, :admin, 404, 200, 0],
+ ].each do |user, token, expect_get_response, expect_list_response, expect_list_items|
+ test "using '#{user}', get '#{token}' by uuid" do
authorize_with user
- get :show, {id: api_client_authorizations(token).uuid}
- assert_response status
+ get :show, {
+ id: api_client_authorizations(token).uuid,
+ }
+ assert_response expect_get_response
+ end
+
+ test "using '#{user}', update '#{token}' by uuid" do
+ authorize_with user
+ put :update, {
+ id: api_client_authorizations(token).uuid,
+ api_client_authorization: {},
+ }
+ assert_response expect_get_response
+ end
+
+ test "using '#{user}', delete '#{token}' by uuid" do
+ authorize_with user
+ post :destroy, {
+ id: api_client_authorizations(token).uuid,
+ }
+ assert_response expect_get_response
end
- end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 403], # this belongs to the user of current session, but we can't list it by uuid
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} list #{token} token using uuid and expect #{status}" do
+ test "using '#{user}', list '#{token}' by uuid" do
authorize_with user
get :index, {
- filters: [['uuid','=',api_client_authorizations(token).uuid]]
+ filters: [['uuid','=',api_client_authorizations(token).uuid]],
}
- assert_response status
+ assert_response expect_list_response
+ if expect_list_items
+ assert_equal assigns(:objects).length, expect_list_items
+ end
end
- end
- [
- [:admin, :admin, 200],
- [:admin, :active, 403],
- [:admin, :admin_vm, 200], # this belongs to the user of current session, and can be listed by token
- [:admin_trustedclient, :active, 200],
- ].each do |user, token, status|
- test "as user #{user} list #{token} token using token and expect #{status}" do
+ test "using '#{user}', list '#{token}' by token" do
authorize_with user
get :index, {
- filters: [['api_token','=',api_client_authorizations(token).api_token]]
+ filters: [['api_token','=',api_client_authorizations(token).api_token]],
}
- assert_response status
+ assert_response expect_list_response
+ if expect_list_items
+ assert_equal assigns(:objects).length, expect_list_items
+ end
end
end
+
+ test "scoped token cannot change its own scopes" do
+ authorize_with :admin_vm
+ put :update, {
+ id: api_client_authorizations(:admin_vm).uuid,
+ api_client_authorization: {scopes: ['all']},
+ }
+ assert_response 403
+ end
+
+ test "token cannot change its own uuid" do
+ authorize_with :admin
+ put :update, {
+ id: api_client_authorizations(:admin).uuid,
+ api_client_authorization: {uuid: 'zzzzz-gj3su-zzzzzzzzzzzzzzz'},
+ }
+ assert_response 403
+ end
end
end
if perms['can_write']
assert u.can? write: repo['uuid']
- assert_match /RW/, perms['gitolite_permissions']
+ assert_match /RW\+/, perms['gitolite_permissions']
else
refute_match /W/, perms['gitolite_permissions']
end
if perms['can_manage']
assert u.can? manage: repo['uuid']
- assert_match /RW/, perms['gitolite_permissions']
+ assert_match /RW\+/, perms['gitolite_permissions']
end
end
end
end
[
- {cfg: :git_repo_ssh_base, cfgval: "git@example.com:", match: %r"^git@example.com:/"},
- {cfg: :git_repo_ssh_base, cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:/"},
+ {cfg: :git_repo_ssh_base, cfgval: "git@example.com:", match: %r"^git@example.com:"},
+ {cfg: :git_repo_ssh_base, cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:"},
{cfg: :git_repo_ssh_base, cfgval: false, refute: /^git@/ },
- {cfg: :git_repo_https_base, cfgval: "https://example.com/", match: %r"https://example.com/"},
+ {cfg: :git_repo_https_base, cfgval: "https://example.com/", match: %r"^https://example.com/"},
{cfg: :git_repo_https_base, cfgval: true, match: %r"^https://git.zzzzz.arvadosapi.com/"},
{cfg: :git_repo_https_base, cfgval: false, refute: /^http/ },
].each do |expect|
authorize_with :active
get :index
assert_response :success
+ assert_not_empty json_response['items']
json_response['items'].each do |r|
if expect[:refute]
r['clone_urls'].each do |u|
refute_match expect[:refute], u
end
else
- assert r['clone_urls'].any? do |u|
- expect[:prefix].match u
- end
+ assert((r['clone_urls'].any? do |u|
+ expect[:match].match u
+ end),
+ "no match for #{expect[:match]} in #{r['clone_urls'].inspect}")
end
end
end
require 'test_helper'
require 'crunch_dispatch'
+require 'helpers/git_test_helper'
class CrunchDispatchTest < ActiveSupport::TestCase
+ include GitTestHelper
+
test 'choose cheaper nodes first' do
act_as_system_user do
# Replace test fixtures with a set suitable for testing dispatch
end
end
+ test 'override --cgroup-root with CRUNCH_CGROUP_ROOT' do
+ ENV['CRUNCH_CGROUP_ROOT'] = '/path/to/cgroup'
+ Rails.configuration.crunch_job_wrapper = :none
+ act_as_system_user do
+ j = Job.create(repository: 'active/foo',
+ script: 'hash',
+ script_version: '4fe459abe02d9b365932b8f5dc419439ab4e2577',
+ script_parameters: {})
+ ok = false
+ Open3.expects(:popen3).at_least_once.with do |*args|
+ if args.index(j.uuid)
+ ok = ((i = args.index '--cgroup-root') and
+ (args[i+1] == '/path/to/cgroup'))
+ end
+ true
+ end.raises(StandardError.new('all is well'))
+ dispatch = CrunchDispatch.new
+ dispatch.parse_argv ['--jobs']
+ dispatch.refresh_todo
+ dispatch.start_jobs
+ assert ok
+ end
+ end
+
def assert_with_timeout timeout, message
t = 0
while (t += 0.1) < timeout
return f.flock(File::LOCK_EX|File::LOCK_NB)
end
end
+
+ test 'rate limit of partial line segments' do
+ act_as_system_user do
+ Rails.configuration.crunch_log_partial_line_throttle_period = 1
+
+ job = {}
+ job[:bytes_logged] = 0
+ job[:log_throttle_bytes_so_far] = 0
+ job[:log_throttle_lines_so_far] = 0
+ job[:log_throttle_bytes_skipped] = 0
+ job[:log_throttle_is_open] = true
+ job[:log_throttle_partial_line_last_at] = Time.new(0)
+ job[:log_throttle_first_partial_line] = true
+
+ dispatch = CrunchDispatch.new
+
+ line = "first log line"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal true, limit
+ assert_equal "first log line", line
+ assert_equal 1, job[:log_throttle_lines_so_far]
+
+ # first partial line segment is skipped and counted towards skipped lines
+ now = Time.now.strftime('%Y-%m-%d-%H:%M:%S')
+ line = "#{now} localhost 100 0 stderr [...] this is first partial line segment [...]"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal true, limit
+ assert_includes line, "Rate-limiting partial segments of long lines", line
+ assert_equal 2, job[:log_throttle_lines_so_far]
+
+ # next partial line segment within throttle interval is skipped but not counted towards skipped lines
+ line = "#{now} localhost 100 0 stderr [...] second partial line segment within the interval [...]"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal false, limit
+ assert_equal 2, job[:log_throttle_lines_so_far]
+
+ # next partial line after interval is counted towards skipped lines
+ sleep(1)
+ line = "#{now} localhost 100 0 stderr [...] third partial line segment after the interval [...]"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal false, limit
+ assert_equal 3, job[:log_throttle_lines_so_far]
+
+ # this is not a valid line segment
+ line = "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal true, limit
+ assert_equal "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment", line
+ assert_equal 4, job[:log_throttle_lines_so_far]
+
+ # this also is not a valid line segment
+ line = "#{now} localhost 100 0 stderr does not start correctly but ends with [...]"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal true, limit
+ assert_equal "#{now} localhost 100 0 stderr does not start correctly but ends with [...]", line
+ assert_equal 5, job[:log_throttle_lines_so_far]
+ end
+ end
end
package main
import (
+ "io/ioutil"
"os"
"os/exec"
- "io/ioutil"
+ "strings"
check "gopkg.in/check.v1"
)
c.Log(prog, " ", args)
cmd := exec.Command(prog, args...)
cmd.Dir = s.gitoliteHome
- cmd.Env = append(os.Environ(), "HOME=" + s.gitoliteHome)
+ cmd.Env = []string{"HOME=" + s.gitoliteHome}
+ for _, e := range os.Environ() {
+ if !strings.HasPrefix(e, "HOME=") {
+ cmd.Env = append(cmd.Env, e)
+ }
+ }
diags, err := cmd.CombinedOutput()
c.Log(string(diags))
c.Assert(err, check.Equals, nil)
// Check that the commit hash appears in the gitolite log, as
// assurance that the gitolite hooks really did run.
- sha1, err := exec.Command("git", "--git-dir", s.tmpWorkdir + "/.git",
+ sha1, err := exec.Command("git", "--git-dir", s.tmpWorkdir+"/.git",
"log", "-n1", "--format=%H").CombinedOutput()
c.Logf("git-log in workdir: %q", string(sha1))
c.Assert(err, check.Equals, nil)
c.Assert(len(sha1), check.Equals, 41)
- gitoliteLog, err := exec.Command("grep", "-r", string(sha1[:40]), s.gitoliteHome + "/.gitolite/logs").CombinedOutput()
+ gitoliteLog, err := exec.Command("grep", "-r", string(sha1[:40]), s.gitoliteHome+"/.gitolite/logs").CombinedOutput()
c.Check(err, check.Equals, nil)
c.Logf("gitolite log message: %q", string(gitoliteLog))
}
log.Printf("Error updating container state to Complete for %v: %q", uuid, err)
}
}
+
+ log.Printf("Finished container run for %v", uuid)
}
--- /dev/null
+package main
+
+import (
+ "flag"
+ "fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "os/signal"
+ "sync"
+ "syscall"
+ "time"
+)
+
+func main() {
+ err := doMain()
+ if err != nil {
+ log.Fatalf("%q", err)
+ }
+}
+
+var (
+ arv arvadosclient.ArvadosClient
+ runningCmds map[string]*exec.Cmd
+ runningCmdsMutex sync.Mutex
+ waitGroup sync.WaitGroup
+ doneProcessing chan bool
+ sigChan chan os.Signal
+)
+
+func doMain() error {
+ flags := flag.NewFlagSet("crunch-dispatch-slurm", flag.ExitOnError)
+
+ pollInterval := flags.Int(
+ "poll-interval",
+ 10,
+ "Interval in seconds to poll for queued containers")
+
+ priorityPollInterval := flags.Int(
+ "container-priority-poll-interval",
+ 60,
+ "Interval in seconds to check priority of a dispatched container")
+
+ crunchRunCommand := flags.String(
+ "crunch-run-command",
+ "/usr/bin/crunch-run",
+ "Crunch command to run container")
+
+ finishCommand := flags.String(
+ "finish-command",
+ "/usr/bin/crunch-finish-slurm.sh",
+ "Command to run from strigger when job is finished")
+
+ // Parse args; omit the first arg which is the command name
+ flags.Parse(os.Args[1:])
+
+ var err error
+ arv, err = arvadosclient.MakeArvadosClient()
+ if err != nil {
+ return err
+ }
+
+ // Channel to terminate
+ doneProcessing = make(chan bool)
+
+ // Graceful shutdown
+ sigChan = make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+ go func(sig <-chan os.Signal) {
+ for sig := range sig {
+ log.Printf("Caught signal: %v", sig)
+ doneProcessing <- true
+ }
+ }(sigChan)
+
+ // Run all queued containers
+ runQueuedContainers(*pollInterval, *priorityPollInterval, *crunchRunCommand, *finishCommand)
+
+ // Wait for all running crunch jobs to complete / terminate
+ waitGroup.Wait()
+
+ return nil
+}
+
+// Poll for queued containers using pollInterval.
+// Invoke dispatchSlurm for each ticker cycle, which will run all the queued containers.
+//
+// Any errors encountered are logged but the program would continue to run (not exit).
+// This is because, once one or more crunch jobs are running,
+// we would need to wait for them complete.
+func runQueuedContainers(pollInterval, priorityPollInterval int, crunchRunCommand, finishCommand string) {
+ ticker := time.NewTicker(time.Duration(pollInterval) * time.Second)
+
+ for {
+ select {
+ case <-ticker.C:
+ dispatchSlurm(priorityPollInterval, crunchRunCommand, finishCommand)
+ case <-doneProcessing:
+ ticker.Stop()
+ return
+ }
+ }
+}
+
+// Container data
+type Container struct {
+ UUID string `json:"uuid"`
+ State string `json:"state"`
+ Priority int `json:"priority"`
+}
+
+// ContainerList is a list of the containers from api
+type ContainerList struct {
+ Items []Container `json:"items"`
+}
+
+// Get the list of queued containers from API server and invoke run for each container.
+func dispatchSlurm(priorityPollInterval int, crunchRunCommand, finishCommand string) {
+ params := arvadosclient.Dict{
+ "filters": [][]string{[]string{"state", "=", "Queued"}},
+ }
+
+ var containers ContainerList
+ err := arv.List("containers", params, &containers)
+ if err != nil {
+ log.Printf("Error getting list of queued containers: %q", err)
+ return
+ }
+
+ for i := 0; i < len(containers.Items); i++ {
+ log.Printf("About to submit queued container %v", containers.Items[i].UUID)
+ // Run the container
+ go run(containers.Items[i], crunchRunCommand, finishCommand, priorityPollInterval)
+ }
+}
+
+// sbatchCmd
+func sbatchFunc(uuid string) *exec.Cmd {
+ return exec.Command("sbatch", "--job-name="+uuid, "--share", "--parsable")
+}
+
+var sbatchCmd = sbatchFunc
+
+// striggerCmd
+func striggerFunc(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) *exec.Cmd {
+ return exec.Command("strigger", "--set", "--jobid="+jobid, "--fini",
+ fmt.Sprintf("--program=%s %s %s %s %s", finishCommand, apiHost, apiToken, apiInsecure, containerUUID))
+}
+
+var striggerCmd = striggerFunc
+
+// Submit job to slurm using sbatch.
+func submit(container Container, crunchRunCommand string) (jobid string, submitErr error) {
+ submitErr = nil
+
+ // Mark record as complete if anything errors out.
+ defer func() {
+ if submitErr != nil {
+ // This really should be an "Error" state, see #8018
+ updateErr := arv.Update("containers", container.UUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Complete"}},
+ nil)
+ if updateErr != nil {
+ log.Printf("Error updating container state to 'Complete' for %v: %q", container.UUID, updateErr)
+ }
+ }
+ }()
+
+ // Create the command and attach to stdin/stdout
+ cmd := sbatchCmd(container.UUID)
+ stdinWriter, stdinerr := cmd.StdinPipe()
+ if stdinerr != nil {
+ submitErr = fmt.Errorf("Error creating stdin pipe %v: %q", container.UUID, stdinerr)
+ return
+ }
+
+ stdoutReader, stdoutErr := cmd.StdoutPipe()
+ if stdoutErr != nil {
+ submitErr = fmt.Errorf("Error creating stdout pipe %v: %q", container.UUID, stdoutErr)
+ return
+ }
+
+ stderrReader, stderrErr := cmd.StderrPipe()
+ if stderrErr != nil {
+ submitErr = fmt.Errorf("Error creating stderr pipe %v: %q", container.UUID, stderrErr)
+ return
+ }
+
+ err := cmd.Start()
+ if err != nil {
+ submitErr = fmt.Errorf("Error starting %v: %v", cmd.Args, err)
+ return
+ }
+
+ stdoutChan := make(chan []byte)
+ go func() {
+ b, _ := ioutil.ReadAll(stdoutReader)
+ stdoutChan <- b
+ close(stdoutChan)
+ }()
+
+ stderrChan := make(chan []byte)
+ go func() {
+ b, _ := ioutil.ReadAll(stderrReader)
+ stderrChan <- b
+ close(stderrChan)
+ }()
+
+ // Send a tiny script on stdin to execute the crunch-run command
+ // slurm actually enforces that this must be a #! script
+ fmt.Fprintf(stdinWriter, "#!/bin/sh\nexec '%s' '%s'\n", crunchRunCommand, container.UUID)
+ stdinWriter.Close()
+
+ err = cmd.Wait()
+
+ stdoutMsg := <-stdoutChan
+ stderrmsg := <-stderrChan
+
+ if err != nil {
+ submitErr = fmt.Errorf("Container submission failed %v: %v %v", cmd.Args, err, stderrmsg)
+ return
+ }
+
+ // If everything worked out, got the jobid on stdout
+ jobid = string(stdoutMsg)
+
+ return
+}
+
+// finalizeRecordOnFinish uses 'strigger' command to register a script that will run on
+// the slurm controller when the job finishes.
+func finalizeRecordOnFinish(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) {
+ cmd := striggerCmd(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure)
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ log.Printf("While setting up strigger: %v", err)
+ }
+}
+
+// Run a queued container.
+// Set container state to locked (TBD)
+// Submit job to slurm to execute crunch-run command for the container
+// If the container priority becomes zero while crunch job is still running, cancel the job.
+func run(container Container, crunchRunCommand, finishCommand string, priorityPollInterval int) {
+
+ jobid, err := submit(container, crunchRunCommand)
+ if err != nil {
+ log.Printf("Error queuing container run: %v", err)
+ return
+ }
+
+ insecure := "0"
+ if arv.ApiInsecure {
+ insecure = "1"
+ }
+ finalizeRecordOnFinish(jobid, container.UUID, finishCommand, arv.ApiServer, arv.ApiToken, insecure)
+
+ // Update container status to Running, this is a temporary workaround
+ // to avoid resubmitting queued containers because record locking isn't
+ // implemented yet.
+ err = arv.Update("containers", container.UUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Running"}},
+ nil)
+ if err != nil {
+ log.Printf("Error updating container state to 'Running' for %v: %q", container.UUID, err)
+ }
+
+ log.Printf("Submitted container run for %v", container.UUID)
+
+ containerUUID := container.UUID
+
+ // A goroutine to terminate the runner if container priority becomes zero
+ priorityTicker := time.NewTicker(time.Duration(priorityPollInterval) * time.Second)
+ go func() {
+ for _ = range priorityTicker.C {
+ var container Container
+ err := arv.Get("containers", containerUUID, nil, &container)
+ if err != nil {
+ log.Printf("Error getting container info for %v: %q", container.UUID, err)
+ } else {
+ if container.Priority == 0 {
+ log.Printf("Canceling container %v", container.UUID)
+ priorityTicker.Stop()
+ cancelcmd := exec.Command("scancel", "--name="+container.UUID)
+ cancelcmd.Run()
+ }
+ if container.State == "Complete" {
+ priorityTicker.Stop()
+ }
+ }
+ }
+ }()
+
+}
--- /dev/null
+package main
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+
+ "io/ioutil"
+ "log"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+var _ = Suite(&TestSuite{})
+var _ = Suite(&MockArvadosServerSuite{})
+
+type TestSuite struct{}
+type MockArvadosServerSuite struct{}
+
+var initialArgs []string
+
+func (s *TestSuite) SetUpSuite(c *C) {
+ initialArgs = os.Args
+ arvadostest.StartAPI()
+}
+
+func (s *TestSuite) TearDownSuite(c *C) {
+ arvadostest.StopAPI()
+}
+
+func (s *TestSuite) SetUpTest(c *C) {
+ args := []string{"crunch-dispatch-slurm"}
+ os.Args = args
+
+ var err error
+ arv, err = arvadosclient.MakeArvadosClient()
+ if err != nil {
+ c.Fatalf("Error making arvados client: %s", err)
+ }
+}
+
+func (s *TestSuite) TearDownTest(c *C) {
+ arvadostest.ResetEnv()
+ os.Args = initialArgs
+}
+
+func (s *MockArvadosServerSuite) TearDownTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *TestSuite) Test_doMain(c *C) {
+ args := []string{"-poll-interval", "2", "-container-priority-poll-interval", "1", "-crunch-run-command", "echo"}
+ os.Args = append(os.Args, args...)
+
+ var sbatchCmdLine []string
+ var striggerCmdLine []string
+
+ // Override sbatchCmd
+ defer func(orig func(string) *exec.Cmd) {
+ sbatchCmd = orig
+ }(sbatchCmd)
+ sbatchCmd = func(uuid string) *exec.Cmd {
+ sbatchCmdLine = sbatchFunc(uuid).Args
+ return exec.Command("echo", uuid)
+ }
+
+ // Override striggerCmd
+ defer func(orig func(jobid, containerUUID, finishCommand,
+ apiHost, apiToken, apiInsecure string) *exec.Cmd) {
+ striggerCmd = orig
+ }(striggerCmd)
+ striggerCmd = func(jobid, containerUUID, finishCommand, apiHost, apiToken, apiInsecure string) *exec.Cmd {
+ striggerCmdLine = striggerFunc(jobid, containerUUID, finishCommand,
+ apiHost, apiToken, apiInsecure).Args
+ go func() {
+ time.Sleep(5 * time.Second)
+ arv.Update("containers", containerUUID,
+ arvadosclient.Dict{
+ "container": arvadosclient.Dict{"state": "Complete"}},
+ nil)
+ }()
+ return exec.Command("echo", "strigger")
+ }
+
+ go func() {
+ time.Sleep(8 * time.Second)
+ sigChan <- syscall.SIGINT
+ }()
+
+ // There should be no queued containers now
+ params := arvadosclient.Dict{
+ "filters": [][]string{[]string{"state", "=", "Queued"}},
+ }
+ var containers ContainerList
+ err := arv.List("containers", params, &containers)
+ c.Check(err, IsNil)
+ c.Check(len(containers.Items), Equals, 1)
+
+ err = doMain()
+ c.Check(err, IsNil)
+
+ c.Check(sbatchCmdLine, DeepEquals, []string{"sbatch", "--job-name=zzzzz-dz642-queuedcontainer", "--share", "--parsable"})
+ c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini",
+ "--program=/usr/bin/crunch-finish-slurm.sh " + os.Getenv("ARVADOS_API_HOST") + " 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h 1 zzzzz-dz642-queuedcontainer"})
+
+ // There should be no queued containers now
+ err = arv.List("containers", params, &containers)
+ c.Check(err, IsNil)
+ c.Check(len(containers.Items), Equals, 0)
+
+ // Previously "Queued" container should now be in "Complete" state
+ var container Container
+ err = arv.Get("containers", "zzzzz-dz642-queuedcontainer", nil, &container)
+ c.Check(err, IsNil)
+ c.Check(container.State, Equals, "Complete")
+}
+
+func (s *MockArvadosServerSuite) Test_APIErrorGettingContainers(c *C) {
+ apiStubResponses := make(map[string]arvadostest.StubResponse)
+ apiStubResponses["/arvados/v1/containers"] = arvadostest.StubResponse{500, string(`{}`)}
+
+ testWithServerStub(c, apiStubResponses, "echo", "Error getting list of queued containers")
+}
+
+func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubResponse, crunchCmd string, expected string) {
+ apiStub := arvadostest.ServerStub{apiStubResponses}
+
+ api := httptest.NewServer(&apiStub)
+ defer api.Close()
+
+ arv = arvadosclient.ArvadosClient{
+ Scheme: "http",
+ ApiServer: api.URL[7:],
+ ApiToken: "abc123",
+ Client: &http.Client{Transport: &http.Transport{}},
+ Retries: 0,
+ }
+
+ tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
+ c.Check(err, IsNil)
+ defer os.Remove(tempfile.Name())
+ log.SetOutput(tempfile)
+
+ go func() {
+ time.Sleep(2 * time.Second)
+ sigChan <- syscall.SIGTERM
+ }()
+
+ runQueuedContainers(2, 1, crunchCmd, crunchCmd)
+
+ buf, _ := ioutil.ReadFile(tempfile.Name())
+ c.Check(strings.Contains(string(buf), expected), Equals, true)
+}
--- /dev/null
+#!/bin/sh
+
+# Script to be called by strigger when a job finishes. This ensures the job
+# record has the correct state "Complete" even if the node running the job
+# failed.
+
+ARVADOS_API_HOST=$1
+ARVADOS_API_TOKEN=$2
+ARVADOS_API_HOST_INSECURE=$3
+uuid=$4
+jobid=$5
+
+# If it is possible to attach metadata to job records we could look up the
+# above information instead of getting it on the command line. For example,
+# this is the recipe for getting the job name (container uuid) from the job id.
+#uuid=$(squeue --jobs=$jobid --states=all --format=%j --noheader)
+
+export ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE
+
+exec arv container update --uuid $uuid --container '{"state": "Complete"}'
package main
import (
+ "encoding/json"
"errors"
"flag"
+ "fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
"github.com/curoverse/dockerclient"
"io"
+ "io/ioutil"
"log"
"os"
+ "os/exec"
"os/signal"
"strings"
"sync"
"syscall"
+ "time"
)
// IArvadosClient is the minimal Arvados API methods used by crunch-run.
}
// Mount describes the mount points to create inside the container.
-type Mount struct{}
+type Mount struct {
+ Kind string `json:"kind"`
+ Writable bool `json:"writable"`
+ PortableDataHash string `json:"portable_data_hash"`
+ UUID string `json:"uuid"`
+ DeviceType string `json:"device_type"`
+}
// Collection record returned by the API server.
-type Collection struct {
- ManifestText string `json:"manifest_text"`
+type CollectionRecord struct {
+ ManifestText string `json:"manifest_text"`
+ PortableDataHash string `json:"portable_data_hash"`
}
// ContainerRecord is the container record returned by the API server.
Priority int `json:"priority"`
RuntimeConstraints map[string]interface{} `json:"runtime_constraints"`
State string `json:"state"`
+ Output string `json:"output"`
}
// NewLogWriter is a factory function to create a new log writer.
type NewLogWriter func(name string) io.WriteCloser
+type RunArvMount func([]string) (*exec.Cmd, error)
+
+type MkTempDir func(string, string) (string, error)
+
// ThinDockerClient is the minimal Docker client interface used by crunch-run.
type ThinDockerClient interface {
StopContainer(id string, timeout int) error
LoadImage(reader io.Reader) error
CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error)
StartContainer(id string, config *dockerclient.HostConfig) error
- ContainerLogs(id string, options *dockerclient.LogOptions) (io.ReadCloser, error)
+ AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error)
Wait(id string) <-chan dockerclient.WaitResult
RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error)
}
Stderr *ThrottledLogger
LogCollection *CollectionWriter
LogsPDH *string
- CancelLock sync.Mutex
- Cancelled bool
- SigChan chan os.Signal
- finalState string
+ RunArvMount
+ MkTempDir
+ ArvMount *exec.Cmd
+ ArvMountPoint string
+ HostOutputDir string
+ CleanupTempDir []string
+ Binds []string
+ OutputPDH *string
+ CancelLock sync.Mutex
+ Cancelled bool
+ SigChan chan os.Signal
+ ArvMountExit chan error
+ finalState string
}
// SetupSignals sets up signal handling to gracefully terminate the underlying
// Docker container and update state when receiving a TERM, INT or QUIT signal.
-func (runner *ContainerRunner) SetupSignals() error {
+func (runner *ContainerRunner) SetupSignals() {
runner.SigChan = make(chan os.Signal, 1)
signal.Notify(runner.SigChan, syscall.SIGTERM)
signal.Notify(runner.SigChan, syscall.SIGINT)
}
}
}(runner.SigChan)
-
- return nil
}
// LoadImage determines the docker image id from the container record and
runner.CrunchLog.Printf("Fetching Docker image from collection '%s'", runner.ContainerRecord.ContainerImage)
- var collection Collection
+ var collection CollectionRecord
err = runner.ArvClient.Get("collections", runner.ContainerRecord.ContainerImage, nil, &collection)
if err != nil {
- return err
+ return fmt.Errorf("While getting container image collection: %v", err)
}
manifest := manifest.Manifest{Text: collection.ManifestText}
var img, imageID string
for ms := range manifest.StreamIter() {
img = ms.FileStreamSegments[0].Name
if !strings.HasSuffix(img, ".tar") {
- return errors.New("First file in the collection does not end in .tar")
+ return fmt.Errorf("First file in the container image collection does not end in .tar")
}
imageID = img[:len(img)-4]
}
var readCloser io.ReadCloser
readCloser, err = runner.Kc.ManifestFileReader(manifest, img)
if err != nil {
- return err
+ return fmt.Errorf("While creating ManifestFileReader for container image: %v", err)
}
err = runner.Docker.LoadImage(readCloser)
if err != nil {
- return err
+ return fmt.Errorf("While loading container image into Docker: %v", err)
}
} else {
runner.CrunchLog.Print("Docker image is available")
return nil
}
+func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string) (c *exec.Cmd, err error) {
+ c = exec.Command("arv-mount", arvMountCmd...)
+ nt := NewThrottledLogger(runner.NewLogWriter("arv-mount"))
+ c.Stdout = nt
+ c.Stderr = nt
+
+ err = c.Start()
+ if err != nil {
+ return nil, err
+ }
+
+ statReadme := make(chan bool)
+ runner.ArvMountExit = make(chan error)
+
+ keepStatting := true
+ go func() {
+ for keepStatting {
+ time.Sleep(100 * time.Millisecond)
+ _, err = os.Stat(fmt.Sprintf("%s/by_id/README", runner.ArvMountPoint))
+ if err == nil {
+ keepStatting = false
+ statReadme <- true
+ }
+ }
+ close(statReadme)
+ }()
+
+ go func() {
+ runner.ArvMountExit <- c.Wait()
+ close(runner.ArvMountExit)
+ }()
+
+ select {
+ case <-statReadme:
+ break
+ case err := <-runner.ArvMountExit:
+ runner.ArvMount = nil
+ keepStatting = false
+ return nil, err
+ }
+
+ return c, nil
+}
+
+func (runner *ContainerRunner) SetupMounts() (err error) {
+ runner.ArvMountPoint, err = runner.MkTempDir("", "keep")
+ if err != nil {
+ return fmt.Errorf("While creating keep mount temp dir: %v", err)
+ }
+
+ runner.CleanupTempDir = append(runner.CleanupTempDir, runner.ArvMountPoint)
+
+ pdhOnly := true
+ tmpcount := 0
+ arvMountCmd := []string{"--foreground", "--allow-other", "--read-write"}
+ collectionPaths := []string{}
+ runner.Binds = nil
+
+ for bind, mnt := range runner.ContainerRecord.Mounts {
+ if mnt.Kind == "collection" {
+ var src string
+ if mnt.UUID != "" && mnt.PortableDataHash != "" {
+ return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ }
+ if mnt.UUID != "" {
+ if mnt.Writable {
+ return fmt.Errorf("Writing to existing collections currently not permitted.")
+ }
+ pdhOnly = false
+ src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
+ } else if mnt.PortableDataHash != "" {
+ if mnt.Writable {
+ return fmt.Errorf("Can never write to a collection specified by portable data hash")
+ }
+ src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.PortableDataHash)
+ } else {
+ src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount)
+ arvMountCmd = append(arvMountCmd, "--mount-tmp")
+ arvMountCmd = append(arvMountCmd, fmt.Sprintf("tmp%d", tmpcount))
+ tmpcount += 1
+ }
+ if mnt.Writable {
+ if bind == runner.ContainerRecord.OutputPath {
+ runner.HostOutputDir = src
+ }
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
+ } else {
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
+ }
+ collectionPaths = append(collectionPaths, src)
+ } else if mnt.Kind == "tmp" {
+ if bind == runner.ContainerRecord.OutputPath {
+ runner.HostOutputDir, err = runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("While creating mount temp dir: %v", err)
+ }
+ st, staterr := os.Stat(runner.HostOutputDir)
+ if staterr != nil {
+ return fmt.Errorf("While Stat on temp dir: %v", staterr)
+ }
+ err = os.Chmod(runner.HostOutputDir, st.Mode()|os.ModeSetgid|0777)
+ if staterr != nil {
+ return fmt.Errorf("While Chmod temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, runner.HostOutputDir)
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind))
+ } else {
+ runner.Binds = append(runner.Binds, bind)
+ }
+ } else {
+ return fmt.Errorf("Unknown mount kind '%s'", mnt.Kind)
+ }
+ }
+
+ if runner.HostOutputDir == "" {
+ return fmt.Errorf("Output path does not correspond to a writable mount point")
+ }
+
+ if pdhOnly {
+ arvMountCmd = append(arvMountCmd, "--mount-by-pdh", "by_id")
+ } else {
+ arvMountCmd = append(arvMountCmd, "--mount-by-id", "by_id")
+ }
+ arvMountCmd = append(arvMountCmd, runner.ArvMountPoint)
+
+ runner.ArvMount, err = runner.RunArvMount(arvMountCmd)
+ if err != nil {
+ return fmt.Errorf("While trying to start arv-mount: %v", err)
+ }
+
+ for _, p := range collectionPaths {
+ _, err = os.Stat(p)
+ if err != nil {
+ return fmt.Errorf("While checking that input files exist: %v", err)
+ }
+ }
+
+ return nil
+}
+
+func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) {
+ // Handle docker log protocol
+ // https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
+
+ header := make([]byte, 8)
+ for {
+ _, readerr := io.ReadAtLeast(containerReader, header, 8)
+
+ if readerr == nil {
+ readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
+ if header[0] == 1 {
+ // stdout
+ _, readerr = io.CopyN(runner.Stdout, containerReader, readsize)
+ } else {
+ // stderr
+ _, readerr = io.CopyN(runner.Stderr, containerReader, readsize)
+ }
+ }
+
+ if readerr != nil {
+ if readerr != io.EOF {
+ runner.CrunchLog.Printf("While reading docker logs: %v", readerr)
+ }
+
+ closeerr := runner.Stdout.Close()
+ if closeerr != nil {
+ runner.CrunchLog.Printf("While closing stdout logs: %v", closeerr)
+ }
+
+ closeerr = runner.Stderr.Close()
+ if closeerr != nil {
+ runner.CrunchLog.Printf("While closing stderr logs: %v", closeerr)
+ }
+
+ runner.loggingDone <- true
+ close(runner.loggingDone)
+ return
+ }
+ }
+}
+
+// AttachLogs connects the docker container stdout and stderr logs to the
+// Arvados logger which logs to Keep and the API server logs table.
+func (runner *ContainerRunner) AttachStreams() (err error) {
+
+ runner.CrunchLog.Print("Attaching container streams")
+
+ var containerReader io.Reader
+ containerReader, err = runner.Docker.AttachContainer(runner.ContainerID,
+ &dockerclient.AttachOptions{Stream: true, Stdout: true, Stderr: true})
+ if err != nil {
+ return fmt.Errorf("While attaching container stdout/stderr streams: %v", err)
+ }
+
+ runner.loggingDone = make(chan bool)
+
+ runner.Stdout = NewThrottledLogger(runner.NewLogWriter("stdout"))
+ runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr"))
+
+ go runner.ProcessDockerAttach(containerReader)
+
+ return nil
+}
+
// StartContainer creates the container and runs it.
func (runner *ContainerRunner) StartContainer() (err error) {
runner.CrunchLog.Print("Creating Docker container")
for k, v := range runner.ContainerRecord.Environment {
runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v)
}
+ runner.ContainerConfig.NetworkDisabled = true
runner.ContainerID, err = runner.Docker.CreateContainer(&runner.ContainerConfig, "", nil)
if err != nil {
- return
+ return fmt.Errorf("While creating container: %v", err)
}
- hostConfig := &dockerclient.HostConfig{}
+ hostConfig := &dockerclient.HostConfig{Binds: runner.Binds,
+ LogConfig: dockerclient.LogConfig{Type: "none"}}
- runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
- err = runner.Docker.StartContainer(runner.ContainerID, hostConfig)
+ err = runner.AttachStreams()
if err != nil {
- return
+ return err
}
- return nil
-}
-
-// AttachLogs connects the docker container stdout and stderr logs to the
-// Arvados logger which logs to Keep and the API server logs table.
-func (runner *ContainerRunner) AttachLogs() (err error) {
-
- runner.CrunchLog.Print("Attaching container logs")
-
- var stderrReader, stdoutReader io.Reader
- stderrReader, err = runner.Docker.ContainerLogs(runner.ContainerID, &dockerclient.LogOptions{Follow: true, Stderr: true})
- if err != nil {
- return
- }
- stdoutReader, err = runner.Docker.ContainerLogs(runner.ContainerID, &dockerclient.LogOptions{Follow: true, Stdout: true})
+ runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
+ err = runner.Docker.StartContainer(runner.ContainerID, hostConfig)
if err != nil {
- return
+ return fmt.Errorf("While starting container: %v", err)
}
- runner.loggingDone = make(chan bool)
-
- runner.Stdout = NewThrottledLogger(runner.NewLogWriter("stdout"))
- runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr"))
- go ReadWriteLines(stdoutReader, runner.Stdout, runner.loggingDone)
- go ReadWriteLines(stderrReader, runner.Stderr, runner.loggingDone)
-
return nil
}
// WaitFinish waits for the container to terminate, capture the exit code, and
// close the stdout/stderr logging.
func (runner *ContainerRunner) WaitFinish() error {
+ runner.CrunchLog.Print("Waiting for container to finish")
+
result := runner.Docker.Wait(runner.ContainerID)
wr := <-result
if wr.Error != nil {
- return wr.Error
+ return fmt.Errorf("While waiting for container to finish: %v", wr.Error)
}
runner.ExitCode = &wr.ExitCode
- // drain stdout/stderr
- <-runner.loggingDone
+ // wait for stdout/stderr to complete
<-runner.loggingDone
- runner.Stdout.Close()
- runner.Stderr.Close()
+ return nil
+}
+
+// HandleOutput sets the output, unmounts the FUSE mount, and deletes temporary directories
+func (runner *ContainerRunner) CaptureOutput() error {
+ if runner.finalState != "Complete" {
+ return nil
+ }
+
+ if runner.HostOutputDir == "" {
+ return nil
+ }
+
+ _, err := os.Stat(runner.HostOutputDir)
+ if err != nil {
+ return fmt.Errorf("While checking host output path: %v", err)
+ }
+
+ var manifestText string
+
+ collectionMetafile := fmt.Sprintf("%s/.arvados#collection", runner.HostOutputDir)
+ _, err = os.Stat(collectionMetafile)
+ if err != nil {
+ // Regular directory
+ cw := CollectionWriter{runner.Kc, nil, sync.Mutex{}}
+ manifestText, err = cw.WriteTree(runner.HostOutputDir, runner.CrunchLog.Logger)
+ if err != nil {
+ return fmt.Errorf("While uploading output files: %v", err)
+ }
+ } else {
+ // FUSE mount directory
+ file, openerr := os.Open(collectionMetafile)
+ if openerr != nil {
+ return fmt.Errorf("While opening FUSE metafile: %v", err)
+ }
+ defer file.Close()
+
+ rec := CollectionRecord{}
+ err = json.NewDecoder(file).Decode(&rec)
+ if err != nil {
+ return fmt.Errorf("While reading FUSE metafile: %v", err)
+ }
+ manifestText = rec.ManifestText
+ }
+
+ var response CollectionRecord
+ err = runner.ArvClient.Create("collections",
+ arvadosclient.Dict{
+ "collection": arvadosclient.Dict{
+ "manifest_text": manifestText}},
+ &response)
+ if err != nil {
+ return fmt.Errorf("While creating output collection: %v", err)
+ }
+
+ runner.OutputPDH = new(string)
+ *runner.OutputPDH = response.PortableDataHash
return nil
}
+func (runner *ContainerRunner) CleanupDirs() {
+ if runner.ArvMount != nil {
+ umount := exec.Command("fusermount", "-z", "-u", runner.ArvMountPoint)
+ umnterr := umount.Run()
+ if umnterr != nil {
+ runner.CrunchLog.Printf("While running fusermount: %v", umnterr)
+ }
+
+ mnterr := <-runner.ArvMountExit
+ if mnterr != nil {
+ runner.CrunchLog.Printf("Arv-mount exit error: %v", mnterr)
+ }
+ }
+
+ for _, tmpdir := range runner.CleanupTempDir {
+ rmerr := os.RemoveAll(tmpdir)
+ if rmerr != nil {
+ runner.CrunchLog.Printf("While cleaning up temporary directory %s: %v", tmpdir, rmerr)
+ }
+ }
+}
+
// CommitLogs posts the collection containing the final container logs.
func (runner *ContainerRunner) CommitLogs() error {
runner.CrunchLog.Print(runner.finalState)
mt, err := runner.LogCollection.ManifestText()
if err != nil {
- return err
+ return fmt.Errorf("While creating log manifest: %v", err)
}
- response := make(map[string]string)
+ var response CollectionRecord
err = runner.ArvClient.Create("collections",
- arvadosclient.Dict{"name": "logs for " + runner.ContainerRecord.UUID,
- "manifest_text": mt},
- response)
+ arvadosclient.Dict{
+ "collection": arvadosclient.Dict{
+ "name": "logs for " + runner.ContainerRecord.UUID,
+ "manifest_text": mt}},
+ &response)
if err != nil {
- return err
+ return fmt.Errorf("While creating log collection: %v", err)
}
runner.LogsPDH = new(string)
- *runner.LogsPDH = response["portable_data_hash"]
+ *runner.LogsPDH = response.PortableDataHash
return nil
}
// UpdateContainerRecordRunning updates the container state to "Running"
func (runner *ContainerRunner) UpdateContainerRecordRunning() error {
- update := arvadosclient.Dict{"state": "Running"}
- return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID, update, nil)
+ return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID,
+ arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running"}}, nil)
}
// UpdateContainerRecordComplete updates the container record state on API
if runner.ExitCode != nil {
update["exit_code"] = *runner.ExitCode
}
+ if runner.OutputPDH != nil {
+ update["output"] = runner.OutputPDH
+ }
update["state"] = runner.finalState
- return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID, update, nil)
+ return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID, arvadosclient.Dict{"container": update}, nil)
}
// NewArvLogWriter creates an ArvLogWriter
}
// Run the full container lifecycle.
-func (runner *ContainerRunner) Run(containerUUID string) (err error) {
- runner.CrunchLog.Printf("Executing container '%s'", containerUUID)
+func (runner *ContainerRunner) Run() (err error) {
+ runner.CrunchLog.Printf("Executing container '%s'", runner.ContainerRecord.UUID)
+
+ hostname, hosterr := os.Hostname()
+ if hosterr != nil {
+ runner.CrunchLog.Printf("Error getting hostname '%v'", hosterr)
+ } else {
+ runner.CrunchLog.Printf("Executing on host '%s'", hostname)
+ }
var runerr, waiterr error
runner.finalState = "Complete"
}
- // (6) write logs
+ // (6) capture output
+ outputerr := runner.CaptureOutput()
+ if outputerr != nil {
+ runner.CrunchLog.Print(outputerr)
+ }
+
+ // (7) clean up temporary directories
+ runner.CleanupDirs()
+
+ // (8) write logs
logerr := runner.CommitLogs()
if logerr != nil {
runner.CrunchLog.Print(logerr)
}
- // (7) update container record with results
+ // (9) update container record with results
updateerr := runner.UpdateContainerRecordComplete()
if updateerr != nil {
runner.CrunchLog.Print(updateerr)
if runerr != nil {
err = runerr
} else if waiterr != nil {
- err = runerr
+ err = waiterr
} else if logerr != nil {
err = logerr
} else if updateerr != nil {
}
}()
- err = runner.ArvClient.Get("containers", containerUUID, nil, &runner.ContainerRecord)
+ err = runner.ArvClient.Get("containers", runner.ContainerRecord.UUID, nil, &runner.ContainerRecord)
if err != nil {
- return
+ return fmt.Errorf("While getting container record: %v", err)
}
- // (0) setup signal handling
- err = runner.SetupSignals()
+ // (1) setup signal handling
+ runner.SetupSignals()
+
+ // (2) check for and/or load image
+ err = runner.LoadImage()
if err != nil {
- return
+ return fmt.Errorf("While loading container image: %v", err)
}
- // (1) check for and/or load image
- err = runner.LoadImage()
+ // (3) set up FUSE mount and binds
+ err = runner.SetupMounts()
if err != nil {
- return
+ return fmt.Errorf("While setting up mounts: %v", err)
}
- // (2) start container
+ // (3) create and start container
err = runner.StartContainer()
if err != nil {
if err == ErrCancelled {
return
}
- // (3) update container record state
+ // (4) update container record state
err = runner.UpdateContainerRecordRunning()
if err != nil {
runner.CrunchLog.Print(err)
}
- // (4) attach container logs
- runerr = runner.AttachLogs()
- if runerr != nil {
- runner.CrunchLog.Print(runerr)
- }
-
// (5) wait for container to finish
waiterr = runner.WaitFinish()
// NewContainerRunner creates a new container runner.
func NewContainerRunner(api IArvadosClient,
kc IKeepClient,
- docker ThinDockerClient) *ContainerRunner {
+ docker ThinDockerClient,
+ containerUUID string) *ContainerRunner {
cr := &ContainerRunner{ArvClient: api, Kc: kc, Docker: docker}
cr.NewLogWriter = cr.NewArvLogWriter
- cr.LogCollection = &CollectionWriter{kc, nil}
+ cr.RunArvMount = cr.ArvMountCmd
+ cr.MkTempDir = ioutil.TempDir
+ cr.LogCollection = &CollectionWriter{kc, nil, sync.Mutex{}}
+ cr.ContainerRecord.UUID = containerUUID
cr.CrunchLog = NewThrottledLogger(cr.NewLogWriter("crunch-run"))
+ cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
return cr
}
func main() {
flag.Parse()
+ containerId := flag.Arg(0)
+
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Fatal(err)
+ log.Fatalf("%s: %v", containerId, err)
}
api.Retries = 8
var kc *keepclient.KeepClient
kc, err = keepclient.MakeKeepClient(&api)
if err != nil {
- log.Fatal(err)
+ log.Fatalf("%s: %v", containerId, err)
}
kc.Retries = 4
var docker *dockerclient.DockerClient
docker, err = dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
if err != nil {
- log.Fatal(err)
+ log.Fatalf("%s: %v", containerId, err)
}
- cr := NewContainerRunner(api, kc, docker)
+ cr := NewContainerRunner(api, kc, docker, containerId)
- err = cr.Run(flag.Arg(0))
+ err = cr.Run()
if err != nil {
- log.Fatal(err)
+ log.Fatalf("%s: %v", containerId, err)
}
}
. "gopkg.in/check.v1"
"io"
"io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "sort"
"strings"
"syscall"
"testing"
type ArvTestClient struct {
Total int64
Calls int
- Content arvadosclient.Dict
+ Content []arvadosclient.Dict
ContainerRecord
Logs map[string]*bytes.Buffer
WasSetRunning bool
var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
type TestDockerClient struct {
- imageLoaded string
- stdoutReader io.ReadCloser
- stderrReader io.ReadCloser
- stdoutWriter io.WriteCloser
- stderrWriter io.WriteCloser
- fn func(t *TestDockerClient)
- finish chan dockerclient.WaitResult
- stop chan bool
- cwd string
- env []string
+ imageLoaded string
+ logReader io.ReadCloser
+ logWriter io.WriteCloser
+ fn func(t *TestDockerClient)
+ finish chan dockerclient.WaitResult
+ stop chan bool
+ cwd string
+ env []string
}
func NewTestDockerClient() *TestDockerClient {
t := &TestDockerClient{}
- t.stdoutReader, t.stdoutWriter = io.Pipe()
- t.stderrReader, t.stderrWriter = io.Pipe()
+ t.logReader, t.logWriter = io.Pipe()
t.finish = make(chan dockerclient.WaitResult)
t.stop = make(chan bool)
t.cwd = "/"
}
}
-func (t *TestDockerClient) ContainerLogs(id string, options *dockerclient.LogOptions) (io.ReadCloser, error) {
- if options.Stdout {
- return t.stdoutReader, nil
- }
- if options.Stderr {
- return t.stderrReader, nil
- }
- return nil, nil
+func (t *TestDockerClient) AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error) {
+ return t.logReader, nil
}
func (t *TestDockerClient) Wait(id string) <-chan dockerclient.WaitResult {
output interface{}) error {
this.Calls += 1
- this.Content = parameters
+ this.Content = append(this.Content, parameters)
if resourceType == "logs" {
- et := parameters["event_type"].(string)
+ et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
if this.Logs == nil {
this.Logs = make(map[string]*bytes.Buffer)
}
if this.Logs[et] == nil {
this.Logs[et] = &bytes.Buffer{}
}
- this.Logs[et].Write([]byte(parameters["properties"].(map[string]string)["text"]))
+ this.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
}
if resourceType == "collections" && output != nil {
- mt := parameters["manifest_text"].(string)
- outmap := output.(map[string]string)
- outmap["portable_data_hash"] = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
+ mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
+ outmap := output.(*CollectionRecord)
+ outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
}
return nil
func (this *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
if resourceType == "collections" {
if uuid == hwPDH {
- output.(*Collection).ManifestText = hwManifest
+ output.(*CollectionRecord).ManifestText = hwManifest
} else if uuid == otherPDH {
- output.(*Collection).ManifestText = otherManifest
+ output.(*CollectionRecord).ManifestText = otherManifest
}
}
if resourceType == "containers" {
}
func (this *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
-
- this.Content = parameters
+ this.Calls += 1
+ this.Content = append(this.Content, parameters)
if resourceType == "containers" {
- if parameters["state"] == "Running" {
+ if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
this.WasSetRunning = true
}
func (s *TestSuite) TestLoadImage(c *C) {
kc := &KeepTestClient{}
docker := NewTestDockerClient()
- cr := NewContainerRunner(&ArvTestClient{}, kc, docker)
+ cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
_, err := cr.Docker.RemoveImage(hwImageId, true)
}
func (this KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
- return "", 0, nil
+ return "", 0, errors.New("KeepError")
}
func (this KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.ReadCloserWithLen, error) {
func (s *TestSuite) TestLoadImageArvError(c *C) {
// (1) Arvados error
- cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil)
+ cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.ContainerRecord.ContainerImage = hwPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "ArvError")
+ c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
}
func (s *TestSuite) TestLoadImageKeepError(c *C) {
// (2) Keep error
docker := NewTestDockerClient()
- cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker)
+ cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.ContainerRecord.ContainerImage = hwPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "KeepError")
+ c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
}
func (s *TestSuite) TestLoadImageCollectionError(c *C) {
// (3) Collection doesn't contain image
- cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil)
+ cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.ContainerRecord.ContainerImage = otherPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "First file in the collection does not end in .tar")
+ c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
}
func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
// (4) Collection doesn't contain image
docker := NewTestDockerClient()
- cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker)
+ cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.ContainerRecord.ContainerImage = hwPDH
err := cr.LoadImage()
return nil
}
+func dockerLog(fd byte, msg string) []byte {
+ by := []byte(msg)
+ header := make([]byte, 8+len(by))
+ header[0] = fd
+ header[7] = byte(len(by))
+ copy(header[8:], by)
+ return header
+}
+
func (s *TestSuite) TestRunContainer(c *C) {
docker := NewTestDockerClient()
docker.fn = func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte("Hello world\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, "Hello world\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{}
}
- cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker)
+ cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
var logs TestLogs
cr.NewLogWriter = logs.NewTestLoggingWriter
err = cr.StartContainer()
c.Check(err, IsNil)
- err = cr.AttachLogs()
- c.Check(err, IsNil)
-
err = cr.WaitFinish()
c.Check(err, IsNil)
func (s *TestSuite) TestCommitLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
- cr.ContainerRecord.UUID = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
cr.CrunchLog.Print("Hello world!")
err := cr.CommitLogs()
c.Check(err, IsNil)
- c.Check(api.Content["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- c.Check(api.Content["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
+ c.Check(api.Calls, Equals, 2)
+ c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
}
func (s *TestSuite) TestUpdateContainerRecordRunning(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
- cr.ContainerRecord.UUID = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
err := cr.UpdateContainerRecordRunning()
c.Check(err, IsNil)
- c.Check(api.Content["state"], Equals, "Running")
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
}
func (s *TestSuite) TestUpdateContainerRecordComplete(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
- cr.ContainerRecord.UUID = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.LogsPDH = new(string)
*cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
err := cr.UpdateContainerRecordComplete()
c.Check(err, IsNil)
- c.Check(api.Content["log"], Equals, *cr.LogsPDH)
- c.Check(api.Content["exit_code"], Equals, *cr.ExitCode)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
}
func (s *TestSuite) TestUpdateContainerRecordCancelled(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
- cr.ContainerRecord.UUID = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Cancelled = true
cr.finalState = "Cancelled"
err := cr.UpdateContainerRecordComplete()
c.Check(err, IsNil)
- c.Check(api.Content["log"], IsNil)
- c.Check(api.Content["exit_code"], IsNil)
- c.Check(api.Content["state"], Equals, "Cancelled")
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
+ c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
}
// Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
docker.RemoveImage(hwImageId, true)
api = &ArvTestClient{ContainerRecord: rec}
- cr = NewContainerRunner(api, &KeepTestClient{}, docker)
+ cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ am := &ArvMountCmdLine{}
+ cr.RunArvMount = am.ArvMountTest
- err = cr.Run("zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ err = cr.Run()
c.Check(err, IsNil)
c.Check(api.WasSetRunning, Equals, true)
- c.Check(api.Content["log"], NotNil)
+ c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
if err != nil {
for k, v := range api.Logs {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
}`, func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte("hello world\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, "hello world\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{}
})
- c.Check(api.Content["exit_code"], Equals, 0)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Calls, Equals, 7)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["exit_code"], Equals, 0)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
}`, func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte("hello\n"))
- t.stderrWriter.Write([]byte("world\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, "hello\n"))
+ t.logWriter.Write(dockerLog(2, "world\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 1}
})
- c.Check(api.Content["log"], NotNil)
- c.Check(api.Content["exit_code"], Equals, 1)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Calls, Equals, 8)
+ c.Check(api.Content[7]["container"].(arvadosclient.Dict)["log"], NotNil)
+ c.Check(api.Content[7]["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
+ c.Check(api.Content[7]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
}`, func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte(t.cwd + "\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
- c.Check(api.Content["exit_code"], Equals, 0)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Calls, Equals, 7)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["exit_code"], Equals, 0)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
+
+ log.Print(api.Logs["stdout"].String())
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
}
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
}`, func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte(t.cwd + "\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
- c.Check(api.Content["exit_code"], Equals, 0)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Calls, Equals, 7)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["exit_code"], Equals, 0)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
}
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
docker := NewTestDockerClient()
docker.fn = func(t *TestDockerClient) {
<-t.stop
- t.stdoutWriter.Write([]byte("foo\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, "foo\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
}
docker.RemoveImage(hwImageId, true)
api := &ArvTestClient{ContainerRecord: rec}
- cr := NewContainerRunner(api, &KeepTestClient{}, docker)
+ cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ am := &ArvMountCmdLine{}
+ cr.RunArvMount = am.ArvMountTest
go func() {
for cr.ContainerID == "" {
cr.SigChan <- syscall.SIGINT
}()
- err = cr.Run("zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ err = cr.Run()
c.Check(err, IsNil)
- c.Check(api.Content["log"], NotNil)
+ c.Check(api.Calls, Equals, 6)
+ c.Check(api.Content[5]["container"].(arvadosclient.Dict)["log"], NotNil)
if err != nil {
for k, v := range api.Logs {
}
}
- c.Check(api.Content["state"], Equals, "Cancelled")
+ c.Check(api.Content[5]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
}`, func(t *TestDockerClient) {
- t.stdoutWriter.Write([]byte(t.env[0][7:] + "\n"))
- t.stdoutWriter.Close()
- t.stderrWriter.Close()
+ t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+ t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
- c.Check(api.Content["exit_code"], Equals, 0)
- c.Check(api.Content["state"], Equals, "Complete")
+ c.Check(api.Calls, Equals, 7)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["exit_code"], Equals, 0)
+ c.Check(api.Content[6]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
}
+
+type ArvMountCmdLine struct {
+ Cmd []string
+}
+
+func (am *ArvMountCmdLine) ArvMountTest(c []string) (*exec.Cmd, error) {
+ am.Cmd = c
+ return nil, nil
+}
+
+func (s *TestSuite) TestSetupMounts(c *C) {
+ api := &ArvTestClient{}
+ kc := &KeepTestClient{}
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ am := &ArvMountCmdLine{}
+ cr.RunArvMount = am.ArvMountTest
+
+ i := 0
+ cr.MkTempDir = func(string, string) (string, error) {
+ i += 1
+ d := fmt.Sprintf("/tmp/mktmpdir%d", i)
+ os.Mkdir(d, os.ModePerm)
+ return d, nil
+ }
+
+ {
+ cr.ContainerRecord.Mounts = make(map[string]Mount)
+ cr.ContainerRecord.Mounts["/tmp"] = Mount{Kind: "tmp"}
+ cr.OutputPath = "/tmp"
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir2:/tmp"})
+ cr.CleanupDirs()
+ }
+
+ {
+ i = 0
+ cr.ContainerRecord.Mounts = make(map[string]Mount)
+ cr.ContainerRecord.Mounts["/keeptmp"] = Mount{Kind: "collection", Writable: true}
+ cr.OutputPath = "/keeptmp"
+
+ os.MkdirAll("/tmp/mktmpdir1/tmp0", os.ModePerm)
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir1/tmp0:/keeptmp"})
+ cr.CleanupDirs()
+ }
+
+ {
+ i = 0
+ cr.ContainerRecord.Mounts = make(map[string]Mount)
+ cr.ContainerRecord.Mounts["/keepinp"] = Mount{Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"}
+ cr.ContainerRecord.Mounts["/keepout"] = Mount{Kind: "collection", Writable: true}
+ cr.OutputPath = "/keepout"
+
+ os.MkdirAll("/tmp/mktmpdir1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
+ os.MkdirAll("/tmp/mktmpdir1/tmp0", os.ModePerm)
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ var ss sort.StringSlice = cr.Binds
+ ss.Sort()
+ c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
+ "/tmp/mktmpdir1/tmp0:/keepout"})
+ cr.CleanupDirs()
+ }
+}
stop bool
flusherDone chan bool
Timestamper
+ Immediate *log.Logger
}
// RFC3339Fixed is a fixed-width version of RFC3339 with microsecond precision,
sc := bufio.NewScanner(bytes.NewBuffer(p))
for sc.Scan() {
_, err = fmt.Fprintf(tl.buf, "%s %s\n", now, sc.Text())
+ if tl.Immediate != nil {
+ tl.Immediate.Printf("%s %s\n", now, sc.Text())
+ }
}
return len(p), err
}
}
// write to API
- lr := arvadosclient.Dict{"object_uuid": arvlog.UUID,
- "event_type": arvlog.loggingStream,
- "properties": map[string]string{"text": string(p)}}
+ lr := arvadosclient.Dict{"log": arvadosclient.Dict{
+ "object_uuid": arvlog.UUID,
+ "event_type": arvlog.loggingStream,
+ "properties": map[string]string{"text": string(p)}}}
err2 := arvlog.ArvClient.Create("logs", lr, nil)
if err1 != nil || err2 != nil {
import (
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
. "gopkg.in/check.v1"
"time"
)
func (s *LoggingTestSuite) TestWriteLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
cr.CrunchLog.Print("Hello world!")
logtext := "2015-12-29T15:51:45.000000001Z Hello world!\n" +
"2015-12-29T15:51:45.000000002Z Goodbye\n"
- c.Check(api.Content["event_type"], Equals, "crunch-run")
- c.Check(api.Content["properties"].(map[string]string)["text"], Equals, logtext)
+ c.Check(api.Content[0]["log"].(arvadosclient.Dict)["event_type"], Equals, "crunch-run")
+ c.Check(api.Content[0]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"], Equals, logtext)
c.Check(string(kc.Content), Equals, logtext)
}
func (s *LoggingTestSuite) TestWriteLogsLarge(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
+ cr.CrunchLog.Immediate = nil
for i := 0; i < 2000000; i += 1 {
cr.CrunchLog.Printf("Hello %d", i)
func (s *LoggingTestSuite) TestWriteMultipleLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
- cr := NewContainerRunner(api, kc, nil)
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
ts := &TestTimestamper{}
cr.CrunchLog.Timestamper = ts.Timestamp
stdout := NewThrottledLogger(cr.NewLogWriter("stdout"))
cr.CrunchLog.Close()
logtext1 := "2015-12-29T15:51:45.000000001Z Hello world!\n" +
"2015-12-29T15:51:45.000000003Z Goodbye\n"
- c.Check(api.Content["event_type"], Equals, "crunch-run")
- c.Check(api.Content["properties"].(map[string]string)["text"], Equals, logtext1)
+ c.Check(api.Content[0]["log"].(arvadosclient.Dict)["event_type"], Equals, "crunch-run")
+ c.Check(api.Content[0]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"], Equals, logtext1)
stdout.Close()
logtext2 := "2015-12-29T15:51:45.000000002Z Doing stuff\n" +
"2015-12-29T15:51:45.000000004Z Blurb\n"
- c.Check(api.Content["event_type"], Equals, "stdout")
- c.Check(api.Content["properties"].(map[string]string)["text"], Equals, logtext2)
+ c.Check(api.Content[1]["log"].(arvadosclient.Dict)["event_type"], Equals, "stdout")
+ c.Check(api.Content[1]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"], Equals, logtext2)
mt, err := cr.LogCollection.ManifestText()
c.Check(err, IsNil)
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
"io"
+ "log"
+ "os"
+ "path/filepath"
"strings"
+ "sync"
)
// Block is a data block in a manifest stream
return nil
}
+func (m *CollectionFileWriter) NewFile(fn string) {
+ m.offset += m.length
+ m.length = 0
+ m.fn = fn
+}
+
func (m *CollectionFileWriter) goUpload() {
var errors []error
uploader := m.uploader
finish <- errors
}
-// CollectionWriter makes implements creating new Keep collections by opening files
+// CollectionWriter implements creating new Keep collections by opening files
// and writing to them.
type CollectionWriter struct {
IKeepClient
Streams []*CollectionFileWriter
+ mtx sync.Mutex
}
// Open a new file for writing in the Keep collection.
fn}
go fw.goUpload()
+ m.mtx.Lock()
+ defer m.mtx.Unlock()
m.Streams = append(m.Streams, fw)
return fw
// Finish writing the collection, wait for all blocks to complete uploading.
func (m *CollectionWriter) Finish() error {
var errstring string
+ m.mtx.Lock()
+ defer m.mtx.Unlock()
+
for _, stream := range m.Streams {
if stream.uploader == nil {
continue
var buf bytes.Buffer
+ m.mtx.Lock()
+ defer m.mtx.Unlock()
for _, v := range m.Streams {
+ if len(v.FileStreamSegments) == 0 {
+ continue
+ }
k := v.StreamName
if k == "." {
buf.WriteString(".")
k = strings.Replace(k, "\n", "", -1)
buf.WriteString("./" + k)
}
- for _, b := range v.Blocks {
- buf.WriteString(" ")
- buf.WriteString(b)
+ if len(v.Blocks) > 0 {
+ for _, b := range v.Blocks {
+ buf.WriteString(" ")
+ buf.WriteString(b)
+ }
+ } else {
+ buf.WriteString(" d41d8cd98f00b204e9800998ecf8427e+0")
}
for _, f := range v.FileStreamSegments {
buf.WriteString(" ")
}
return buf.String(), nil
}
+
+type WalkUpload struct {
+ kc IKeepClient
+ stripPrefix string
+ streamMap map[string]*CollectionFileWriter
+ status *log.Logger
+}
+
+// WalkFunc walks a directory tree, uploads each file found and adds it to the
+// CollectionWriter.
+func (m *WalkUpload) WalkFunc(path string, info os.FileInfo, err error) error {
+
+ if info.IsDir() {
+ return nil
+ }
+
+ var dir string
+ if len(path) > (len(m.stripPrefix) + len(info.Name()) + 1) {
+ dir = path[len(m.stripPrefix)+1 : (len(path) - len(info.Name()) - 1)]
+ }
+ if dir == "" {
+ dir = "."
+ }
+
+ fn := path[(len(path) - len(info.Name())):]
+
+ if m.streamMap[dir] == nil {
+ m.streamMap[dir] = &CollectionFileWriter{
+ m.kc,
+ &manifest.ManifestStream{StreamName: dir},
+ 0,
+ 0,
+ nil,
+ make(chan *Block),
+ make(chan []error),
+ ""}
+ go m.streamMap[dir].goUpload()
+ }
+
+ fileWriter := m.streamMap[dir]
+
+ // Reset the CollectionFileWriter for a new file
+ fileWriter.NewFile(fn)
+
+ file, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ m.status.Printf("Uploading %v/%v (%v bytes)", dir, fn, info.Size())
+
+ _, err = io.Copy(fileWriter, file)
+ if err != nil {
+ return err
+ }
+
+ // Commits the current file. Legal to call this repeatedly.
+ fileWriter.Close()
+
+ return nil
+}
+
+func (cw *CollectionWriter) WriteTree(root string, status *log.Logger) (manifest string, err error) {
+ streamMap := make(map[string]*CollectionFileWriter)
+ wu := &WalkUpload{cw.IKeepClient, root, streamMap, status}
+ err = filepath.Walk(root, wu.WalkFunc)
+
+ if err != nil {
+ return "", err
+ }
+
+ cw.mtx.Lock()
+ for _, st := range streamMap {
+ cw.Streams = append(cw.Streams, st)
+ }
+ cw.mtx.Unlock()
+
+ return cw.ManifestText()
+}
--- /dev/null
+package main
+
+import (
+ . "gopkg.in/check.v1"
+ "io/ioutil"
+ "log"
+ "os"
+ "sync"
+)
+
+type UploadTestSuite struct{}
+
+// Gocheck boilerplate
+var _ = Suite(&UploadTestSuite{})
+
+func (s *TestSuite) TestSimpleUpload(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+ c.Check(err, IsNil)
+ c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
+}
+
+func (s *TestSuite) TestSimpleUploadTwofiles(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
+ ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, IsNil)
+ c.Check(str, Equals, ". 3858f62230ac3c915f300c664312c63f+6 0:3:file1.txt 3:3:file2.txt\n")
+}
+
+func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ os.Mkdir(tmpdir+"/subdir", 0700)
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
+ ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, IsNil)
+
+ // streams can get added in either order because of scheduling
+ // of goroutines.
+ if str != `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
+./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
+` && str != `./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
+. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
+` {
+ c.Error("Did not get expected manifest text")
+ }
+}
+
+func (s *TestSuite) TestSimpleUploadLarge(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ file, _ := os.Create(tmpdir + "/" + "file1.txt")
+ data := make([]byte, 1024*1024-1)
+ for i := range data {
+ data[i] = byte(i % 10)
+ }
+ for i := 0; i < 65; i++ {
+ file.Write(data)
+ }
+ file.Close()
+
+ ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, IsNil)
+ c.Check(str, Equals, ". 00ecf01e0d93385115c9f8bed757425d+67108864 485cd630387b6b1846fe429f261ea05f+1048514 0:68157375:file1.txt 68157375:3:file2.txt\n")
+}
+
+func (s *TestSuite) TestUploadEmptySubdir(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ os.Mkdir(tmpdir+"/subdir", 0700)
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, IsNil)
+ c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
+`)
+}
+
+func (s *TestSuite) TestUploadEmptyFile(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
+
+ cw := CollectionWriter{&KeepTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, IsNil)
+ c.Check(str, Equals, `. d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1.txt
+`)
+}
+
+func (s *TestSuite) TestUploadError(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
+
+ cw := CollectionWriter{&KeepErrorTestClient{}, nil, sync.Mutex{}}
+ str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
+
+ c.Check(err, NotNil)
+ c.Check(str, Equals, "")
+}
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
+ collectionBatchSize int
dryRun bool
)
"minutes-between-runs",
0,
"How many minutes we wait between data manager runs. 0 means run once and exit.")
+ flag.IntVar(&collectionBatchSize,
+ "collection-batch-size",
+ 1000,
+ "How many collections to request in each batch.")
flag.BoolVar(&dryRun,
"dry-run",
false,
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
- BatchSize: 50})
+ BatchSize: collectionBatchSize})
collDone <- struct{}{}
}()
Logger: arvLogger,
Limit: 1000})
- <- collDone
+ <-collDone
// Return a nil error only if both parts succeeded.
if collErr != nil {
"net/http"
"os"
"os/exec"
+ "path"
"regexp"
"strings"
"testing"
testOldBlocksNotDeletedOnDataManagerError(t, "", "", false, false)
}
+func createBadPath(t *testing.T) (badpath string) {
+ tempdir, err := ioutil.TempDir("", "bad")
+ if err != nil {
+ t.Fatalf("Could not create temporary directory for bad path: %v", err)
+ }
+ badpath = path.Join(tempdir, "bad")
+ return
+}
+
+func destroyBadPath(t *testing.T, badpath string) {
+ tempdir := path.Join(badpath, "..")
+ err := os.Remove(tempdir)
+ if err != nil {
+ t.Fatalf("Could not remove bad path temporary directory %v: %v", tempdir, err)
+ }
+}
+
func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) {
- testOldBlocksNotDeletedOnDataManagerError(t, "/badwritetofile", "", true, true)
+ badpath := createBadPath(t)
+ defer destroyBadPath(t, badpath)
+ testOldBlocksNotDeletedOnDataManagerError(t, path.Join(badpath, "writetofile"), "", true, true)
}
func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) {
- testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", true, true)
+ badpath := createBadPath(t)
+ defer destroyBadPath(t, badpath)
+ testOldBlocksNotDeletedOnDataManagerError(t, "", path.Join(badpath, "heapprofilefile"), true, true)
}
// Create some blocks and backdate some of them.
def _remove_container(self, cid):
try:
- self.docker_client.remove_container(cid)
+ self.docker_client.remove_container(cid, v=True)
except docker.errors.APIError as error:
logger.warning("Failed to remove container %s: %s", cid, error)
else:
TEST_CLASS = cleaner.DockerImageCleaner
TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True}
+ def test_container_deletion_deletes_volumes(self):
+ cid = MockDockerId()
+ self.events.append(MockEvent('die', docker_id=cid))
+ self.recorder.run()
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
+
@mock.patch('arvados_docker.cleaner.logger')
def test_failed_container_deletion_handling(self, mockLogger):
cid = MockDockerId()
self.docker_client.remove_container.side_effect = MockException(500)
self.events.append(MockEvent('die', docker_id=cid))
self.recorder.run()
- self.docker_client.remove_container.assert_called_with(cid)
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
self.assertEqual("Failed to remove container %s: %s",
mockLogger.warning.call_args[0][0])
self.assertEqual(cid,
def test_remove_onexit(self):
self.args.remove_stopped_containers = 'onexit'
cleaner.run(self.args, self.docker_client)
- self.docker_client.remove_container.assert_called_once_with(self.newCID)
+ self.docker_client.remove_container.assert_called_once_with(self.newCID, v=True)
def test_remove_always(self):
self.args.remove_stopped_containers = 'always'
cleaner.run(self.args, self.docker_client)
- self.docker_client.remove_container.assert_any_call(self.existingCID)
- self.docker_client.remove_container.assert_any_call(self.newCID)
+ self.docker_client.remove_container.assert_any_call(self.existingCID, v=True)
+ self.docker_client.remove_container.assert_any_call(self.newCID, v=True)
self.assertEqual(2, self.docker_client.remove_container.call_count)
def test_remove_never(self):
mock.call.events(since=mock.ANY),
mock.call.containers(filters={'status':'exited'})])
# Asked to delete the container twice?
- self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID)] * 2)
+ self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID, v=True)] * 2)
self.assertEqual(2, self.docker_client.remove_container.call_count)
func (v *AzureBlobVolume) isKeepBlock(s string) bool {
return keepBlockRegexp.MatchString(s)
}
+
+// EmptyTrash looks for trashed blocks that exceeded trashLifetime
+// and deletes them from the volume.
+// TBD
+func (v *AzureBlobVolume) EmptyTrash() {
+}
// trashLifetime is the time duration after a block is trashed
// during which it can be recovered using an /untrash request
+// Use 10s or 10m or 10h to set as 10 seconds or minutes or hours respectively.
var trashLifetime time.Duration
+// trashCheckInterval is the time duration at which the emptyTrash goroutine
+// will check and delete expired trashed blocks. Default is one day.
+// Use 10s or 10m or 10h to set as 10 seconds or minutes or hours respectively.
+var trashCheckInterval time.Duration
+
var maxBuffers = 128
var bufs *bufferPool
&trashLifetime,
"trash-lifetime",
0*time.Second,
- "Interval after a block is trashed during which it can be recovered using an /untrash request")
+ "Time duration after a block is trashed during which it can be recovered using an /untrash request")
+ flag.DurationVar(
+ &trashCheckInterval,
+ "trash-check-interval",
+ 24*time.Hour,
+ "Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day.")
flag.Parse()
trashq = NewWorkQueue()
go RunTrashWorker(trashq)
+ // Start emptyTrash goroutine
+ doneEmptyingTrash := make(chan bool)
+ go emptyTrash(doneEmptyingTrash, trashCheckInterval)
+
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
term := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
s := <-sig
log.Println("caught signal:", s)
+ doneEmptyingTrash <- true
listener.Close()
}(term)
signal.Notify(term, syscall.SIGTERM)
srv := &http.Server{Addr: listen}
srv.Serve(listener)
}
+
+// At every trashCheckInterval tick, invoke EmptyTrash on all volumes.
+func emptyTrash(doneEmptyingTrash chan bool, trashCheckInterval time.Duration) {
+ ticker := time.NewTicker(trashCheckInterval)
+
+ for {
+ select {
+ case <-ticker.C:
+ for _, v := range volumes {
+ if v.Writable() {
+ v.EmptyTrash()
+ }
+ }
+ case <-doneEmptyingTrash:
+ ticker.Stop()
+ return
+ }
+ }
+}
return nil
}
+// Trash a Keep block.
func (v *S3Volume) Trash(loc string) error {
if v.readonly {
return MethodDisabledError
}
return err
}
+
+// EmptyTrash looks for trashed blocks that exceeded trashLifetime
+// and deletes them from the volume.
+// TBD
+func (v *S3Volume) EmptyTrash() {
+}
// underlying device. It will be passed on to clients in
// responses to PUT requests.
Replication() int
+
+ // EmptyTrash looks for trashed blocks that exceeded trashLifetime
+ // and deletes them from the volume.
+ EmptyTrash()
}
// A VolumeManager tells callers which volumes can read, which volumes
testPutFullBlock(t, factory)
testTrashUntrash(t, factory)
+ testTrashEmptyTrashUntrash(t, factory)
}
// Put a test block, get it and verify content
v := factory(t)
defer v.Teardown()
defer func() {
- trashLifetime = 0
+ trashLifetime = 0 * time.Second
}()
trashLifetime = 3600 * time.Second
}
bufs.Put(buf)
}
+
+func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
+ v := factory(t)
+ defer v.Teardown()
+ defer func(orig time.Duration) {
+ trashLifetime = orig
+ }(trashLifetime)
+
+ checkGet := func() error {
+ buf, err := v.Get(TestHash)
+ if err != nil {
+ return err
+ }
+ if bytes.Compare(buf, TestBlock) != 0 {
+ t.Fatalf("Got data %+q, expected %+q", buf, TestBlock)
+ }
+ bufs.Put(buf)
+ return nil
+ }
+
+ // First set: EmptyTrash before reaching the trash deadline.
+
+ trashLifetime = 1 * time.Hour
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ err := checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = v.Trash(TestHash)
+ if err == MethodDisabledError || err == ErrNotImplemented {
+ // Skip the trash tests for read-only volumes, and
+ // volume types that don't support trashLifetime>0.
+ return
+ }
+
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ v.EmptyTrash()
+
+ // Even after emptying the trash, we can untrash our block
+ // because the deadline hasn't been reached.
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Untrash should fail if the only block in the trash has
+ // already been untrashed.
+ err = v.Untrash(TestHash)
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // The failed Untrash should not interfere with our
+ // already-untrashed copy.
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Second set: EmptyTrash after the trash deadline has passed.
+
+ trashLifetime = 1 * time.Nanosecond
+
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Even though 1ns has passed, we can untrash because we
+ // haven't called EmptyTrash yet.
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Trash it again, and this time call EmptyTrash so it really
+ // goes away.
+ err = v.Trash(TestHash)
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Errorf("os.IsNotExist(%v) should have been true", err)
+ }
+ v.EmptyTrash()
+
+ // Untrash won't find it
+ err = v.Untrash(TestHash)
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Get block won't find it
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ // Third set: If the same data block gets written again after
+ // being trashed, and then the trash gets emptied, the newer
+ // un-trashed copy doesn't get deleted along with it.
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Nanosecond
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err == nil || !os.IsNotExist(err) {
+ t.Fatalf("os.IsNotExist(%v) should have been true", err)
+ }
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ // EmptyTrash should not delete the untrashed copy.
+ v.EmptyTrash()
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Fourth set: If the same data block gets trashed twice with
+ // different deadlines A and C, and then the trash is emptied
+ // at intermediate time B (A < B < C), it is still possible to
+ // untrash the block whose deadline is "C".
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Nanosecond
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ v.PutRaw(TestHash, TestBlock)
+ v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
+
+ trashLifetime = time.Hour
+ err = v.Trash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // EmptyTrash should not prevent us from recovering the
+ // time.Hour ("C") trash
+ v.EmptyTrash()
+ err = v.Untrash(TestHash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = checkGet()
+ if err != nil {
+ t.Fatal(err)
+ }
+}
func (v *MockVolume) Replication() int {
return 1
}
+
+func (v *MockVolume) EmptyTrash() {
+}
}
func (vs *unixVolumeAdder) Set(value string) error {
- if trashLifetime != 0 {
- return ErrNotImplemented
- }
if dirs := strings.Split(value, ","); len(dirs) > 1 {
log.Print("DEPRECATED: using comma-separated volume list.")
for _, dir := range dirs {
}
}
-// Delete deletes the block data from the unix storage
+// Trash trashes the block data from the unix storage
+// If trashLifetime == 0, the block is deleted
+// Else, the block is renamed as path/{loc}.trash.{deadline},
+// where deadline = now + trashLifetime
func (v *UnixVolume) Trash(loc string) error {
// Touch() must be called before calling Write() on a block. Touch()
// also uses lockfile(). This avoids a race condition between Write()
- // and Delete() because either (a) the file will be deleted and Touch()
+ // and Trash() because either (a) the file will be trashed and Touch()
// will signal to the caller that the file is not present (and needs to
// be re-written), or (b) Touch() will update the file's timestamp and
- // Delete() will read the correct up-to-date timestamp and choose not to
- // delete the file.
+ // Trash() will read the correct up-to-date timestamp and choose not to
+ // trash the file.
if v.readonly {
return MethodDisabledError
}
- if trashLifetime != 0 {
- return ErrNotImplemented
- }
if v.locker != nil {
v.locker.Lock()
defer v.locker.Unlock()
return nil
}
}
- return os.Remove(p)
+
+ if trashLifetime == 0 {
+ return os.Remove(p)
+ }
+ return os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(trashLifetime).Unix()))
}
// Untrash moves block from trash back into store
-// TBD
-func (v *UnixVolume) Untrash(loc string) error {
- return ErrNotImplemented
+// Look for path/{loc}.trash.{deadline} in storage,
+// and rename the first such file as path/{loc}
+func (v *UnixVolume) Untrash(loc string) (err error) {
+ if v.readonly {
+ return MethodDisabledError
+ }
+
+ files, err := ioutil.ReadDir(v.blockDir(loc))
+ if err != nil {
+ return err
+ }
+
+ if len(files) == 0 {
+ return os.ErrNotExist
+ }
+
+ foundTrash := false
+ prefix := fmt.Sprintf("%v.trash.", loc)
+ for _, f := range files {
+ if strings.HasPrefix(f.Name(), prefix) {
+ foundTrash = true
+ err = os.Rename(v.blockPath(f.Name()), v.blockPath(loc))
+ if err == nil {
+ break
+ }
+ }
+ }
+
+ if foundTrash == false {
+ return os.ErrNotExist
+ }
+
+ return
}
// blockDir returns the fully qualified directory name for the directory
return err
}
}
+
+var trashLocRegexp = regexp.MustCompile(`/([0-9a-f]{32})\.trash\.(\d+)$`)
+
+// EmptyTrash walks hierarchy looking for {hash}.trash.*
+// and deletes those with deadline < now.
+func (v *UnixVolume) EmptyTrash() {
+ var bytesDeleted, bytesInTrash int64
+ var blocksDeleted, blocksInTrash int
+
+ err := filepath.Walk(v.root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ log.Printf("EmptyTrash: filepath.Walk: %v: %v", path, err)
+ return nil
+ }
+ if info.Mode().IsDir() {
+ return nil
+ }
+ matches := trashLocRegexp.FindStringSubmatch(path)
+ if len(matches) != 3 {
+ return nil
+ }
+ deadline, err := strconv.ParseInt(matches[2], 10, 64)
+ if err != nil {
+ log.Printf("EmptyTrash: %v: ParseInt(%v): %v", path, matches[2], err)
+ return nil
+ }
+ bytesInTrash += info.Size()
+ blocksInTrash++
+ if deadline > time.Now().Unix() {
+ return nil
+ }
+ err = os.Remove(path)
+ if err != nil {
+ log.Printf("EmptyTrash: Remove %v: %v", path, err)
+ return nil
+ }
+ bytesDeleted += info.Size()
+ blocksDeleted++
+ return nil
+ })
+
+ if err != nil {
+ log.Printf("EmptyTrash error for %v: %v", v.String(), err)
+ }
+
+ log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+}
--- /dev/null
+from __future__ import absolute_import, print_function
+
+import errno
+import logging
+import os
+import threading
+import traceback
+
+import pykka
+
+class _TellCallableProxy(object):
+ """Internal helper class for proxying callables."""
+
+ def __init__(self, ref, attr_path):
+ self.actor_ref = ref
+ self._attr_path = attr_path
+
+ def __call__(self, *args, **kwargs):
+ message = {
+ 'command': 'pykka_call',
+ 'attr_path': self._attr_path,
+ 'args': args,
+ 'kwargs': kwargs,
+ }
+ self.actor_ref.tell(message)
+
+
+class TellActorProxy(pykka.ActorProxy):
+ """ActorProxy in which all calls are implemented as using tell().
+
+ The standard pykka.ActorProxy always uses ask() and returns a Future. If
+ the target method raises an exception, it is placed in the Future object
+ and re-raised when get() is called on the Future. Unfortunately, most
+ messaging in Node Manager is asynchronous and the caller does not store the
+ Future object returned by the call to ActorProxy. As a result, exceptions
+ resulting from these calls end up in limbo, neither reported in the logs
+ nor handled by on_failure().
+
+ The TellActorProxy uses tell() instead of ask() and does not return a
+ Future object. As a result, if the target method raises an exception, it
+ will be logged and on_failure() will be called as intended.
+
+ """
+
+ def __repr__(self):
+ return '<ActorProxy for %s, attr_path=%s>' % (
+ self.actor_ref, self._attr_path)
+
+ def __getattr__(self, name):
+ """Get a callable from the actor."""
+ attr_path = self._attr_path + (name,)
+ if attr_path not in self._known_attrs:
+ self._known_attrs = self._get_attributes()
+ attr_info = self._known_attrs.get(attr_path)
+ if attr_info is None:
+ raise AttributeError('%s has no attribute "%s"' % (self, name))
+ if attr_info['callable']:
+ if attr_path not in self._callable_proxies:
+ self._callable_proxies[attr_path] = _TellCallableProxy(
+ self.actor_ref, attr_path)
+ return self._callable_proxies[attr_path]
+ else:
+ raise AttributeError('attribute "%s" is not a callable on %s' % (name, self))
+
+class TellableActorRef(pykka.ActorRef):
+ """ActorRef adding the tell_proxy() method to get TellActorProxy."""
+
+ def tell_proxy(self):
+ return TellActorProxy(self)
+
+class BaseNodeManagerActor(pykka.ThreadingActor):
+ """Base class for actors in node manager, redefining actor_ref as a
+ TellableActorRef and providing a default on_failure handler.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(pykka.ThreadingActor, self).__init__(*args, **kwargs)
+ self.actor_ref = TellableActorRef(self)
+
+ def on_failure(self, exception_type, exception_value, tb):
+ lg = getattr(self, "_logger", logging)
+ if (exception_type in (threading.ThreadError, MemoryError) or
+ exception_type is OSError and exception_value.errno == errno.ENOMEM):
+ lg.critical("Unhandled exception is a fatal error, killing Node Manager")
+ os.killpg(os.getpgid(0), 9)
super(RemotePollLoopActor, self).__init__()
self._client = client
self._timer = timer_actor
- self._later = self.actor_ref.proxy()
+ self._later = self.actor_ref.tell_proxy()
self._polling_started = False
self.min_poll_wait = poll_wait
self.max_poll_wait = max_poll_wait
super(ComputeNodeStateChangeBase, self).__init__()
RetryMixin.__init__(self, retry_wait, max_retry_wait,
None, cloud_client, timer_actor)
- self._later = self.actor_ref.proxy()
+ self._later = self.actor_ref.tell_proxy()
self._arvados = arvados_client
self.subscribers = set()
self._set_logger()
def _finished(self):
- _notify_subscribers(self._later, self.subscribers)
+ if self.subscribers is None:
+ raise Exception("Actor tried to finish twice")
+ _notify_subscribers(self.actor_ref.proxy(), self.subscribers)
self.subscribers = None
self._logger.info("finished")
def subscribe(self, subscriber):
if self.subscribers is None:
try:
- subscriber(self._later)
+ subscriber(self.actor_ref.proxy())
except pykka.ActorDeadError:
pass
else:
if not self._cloud.destroy_node(self.cloud_node):
if self._cloud.broken(self.cloud_node):
self._later.cancel_shutdown(self.NODE_BROKEN)
+ return
else:
# Force a retry.
raise cloud_types.LibcloudError("destroy_node failed")
boot_fail_after=1800
):
super(ComputeNodeMonitorActor, self).__init__()
- self._later = self.actor_ref.proxy()
+ self._later = self.actor_ref.tell_proxy()
self._last_log = None
self._shutdowns = shutdown_timer
self._cloud_node_fqdn = cloud_fqdn_func
eligible = self.shutdown_eligible()
if eligible is True:
self._debug("Suggesting shutdown.")
- _notify_subscribers(self._later, self.subscribers)
+ _notify_subscribers(self.actor_ref.proxy(), self.subscribers)
elif self._shutdowns.window_open():
self._debug("Cannot shut down because %s", eligible)
elif self.last_shutdown_opening != next_opening:
first_ping_s = arvados_node.get('first_ping_at')
if (self.arvados_node is not None) or (not first_ping_s):
return None
- elif ((arvados_node['ip_address'] in self.cloud_node.private_ips) and
+ elif ((arvados_node['info'].get('ec2_instance_id') == self._cloud.node_id(self.cloud_node)) and
(arvados_timestamp(first_ping_s) >= self.cloud_node_start_time)):
self._later.update_arvados_node(arvados_node)
return self.cloud_node.id
self.ping_host, arvados_node['uuid'],
arvados_node['info']['ping_secret'])
- def find_node(self, name):
- node = [n for n in self.list_nodes() if n.name == name]
- if node:
- return node[0]
- else:
- return None
+ @staticmethod
+ def _name_key(cloud_object):
+ return cloud_object.name
def create_node(self, size, arvados_node):
try:
# loop forever because subsequent create_node attempts will fail
# due to node name collision. So check if the node we intended to
# create shows up in the cloud node list and return it if found.
- try:
- node = self.find_node(kwargs['name'])
- if node:
- return node
- except:
- # Ignore possible exception from find_node in favor of
- # re-raising the original create_node exception.
- pass
- raise
+ node = self.search_for(kwargs['name'], 'list_nodes', self._name_key)
+ if node:
+ return node
+ else:
+ # something else went wrong, re-raise the exception
+ raise
def post_create_node(self, cloud_node):
# ComputeNodeSetupActor calls this method after the cloud node is
lambda self, value: setattr(self.real, attr_name, value),
doc=getattr(getattr(NodeDriver, attr_name), '__doc__', None))
+ # node id
+ @classmethod
+ def node_id(cls):
+ raise NotImplementedError("BaseComputeNodeDriver.node_id")
+
_locals = locals()
for _attr_name in dir(NodeDriver):
if (not _attr_name.startswith('_')) and (_attr_name not in _locals):
# Azure only supports filtering node lists by resource group.
# Do our own filtering based on tag.
nodes = [node for node in
- super(ComputeNodeDriver, self).list_nodes()
+ super(ComputeNodeDriver, self).list_nodes(ex_fetch_nic=False)
if node.extra["tags"].get("arvados-class") == self.tags["arvados-class"]]
for n in nodes:
# Need to populate Node.size
@classmethod
def node_start_time(cls, node):
return arvados_timestamp(node.extra["tags"].get("booted_at"))
+
+ @classmethod
+ def node_id(cls, node):
+ return node.name
self.real.ex_create_tags(cloud_node,
{'Name': arvados_node_fqdn(arvados_node)})
- def find_node(self, name):
- raise NotImplementedError("ec2.ComputeNodeDriver.find_node")
-
def list_nodes(self):
# Need to populate Node.size
nodes = super(ComputeNodeDriver, self).list_nodes()
time_str = node.extra['launch_time'].split('.', 2)[0] + 'UTC'
return time.mktime(time.strptime(
time_str,'%Y-%m-%dT%H:%M:%S%Z')) - time.timezone
+
+ @classmethod
+ def node_id(cls, node):
+ return node.id
self._disktype_links = {dt.name: self._object_link(dt)
for dt in self.real.ex_list_disktypes()}
- @staticmethod
- def _name_key(cloud_object):
- return cloud_object.name
-
@staticmethod
def _object_link(cloud_object):
return cloud_object.extra.get('selfLink')
node.extra['metadata']['items'], 'booted_at'))
except KeyError:
return 0
+
+ @classmethod
+ def node_id(cls, node):
+ return node.id
import pykka
from apiclient import errors as apierror
+from .baseactor import BaseNodeManagerActor
+
# IOError is the base class for socket.error, ssl.SSLError, and friends.
# It seems like it hits the sweet spot for operations we want to retry:
# it's low-level, but unlikely to catch code bugs.
NETWORK_ERRORS = (IOError,)
ARVADOS_ERRORS = NETWORK_ERRORS + (apierror.Error,)
-actor_class = pykka.ThreadingActor
+actor_class = BaseNodeManagerActor
class NodeManagerConfig(ConfigParser.SafeConfigParser):
"""Node Manager Configuration class.
self._new_arvados = arvados_factory
self._new_cloud = cloud_factory
self._cloud_driver = self._new_cloud()
- self._later = self.actor_ref.proxy()
+ self._later = self.actor_ref.tell_proxy()
self.shutdown_windows = shutdown_windows
self.server_calculator = server_calculator
self.min_cloud_size = self.server_calculator.cheapest_size()
poll_stale_after=self.poll_stale_after,
node_stale_after=self.node_stale_after,
cloud_client=self._cloud_driver,
- boot_fail_after=self.boot_fail_after).proxy()
- actor.subscribe(self._later.node_can_shutdown)
+ boot_fail_after=self.boot_fail_after)
+ actorTell = actor.tell_proxy()
+ actorTell.subscribe(self._later.node_can_shutdown)
self._cloud_nodes_actor.subscribe_to(cloud_node.id,
- actor.update_cloud_node)
- record = _ComputeNodeRecord(actor, cloud_node)
+ actorTell.update_cloud_node)
+ record = _ComputeNodeRecord(actor.proxy(), cloud_node)
return record
def update_cloud_nodes(self, nodelist):
shutdown = self._node_shutdown.start(
timer_actor=self._timer, cloud_client=self._new_cloud(),
arvados_client=self._new_arvados(),
- node_monitor=node_actor.actor_ref, cancellable=cancellable).proxy()
- self.shutdowns[cloud_node_id] = shutdown
+ node_monitor=node_actor.actor_ref, cancellable=cancellable)
+ self.shutdowns[cloud_node_id] = shutdown.proxy()
self.sizes_booting_shutdown[cloud_node_id] = cloud_node_obj.size
- shutdown.subscribe(self._later.node_finished_shutdown)
+ shutdown.tell_proxy().subscribe(self._later.node_finished_shutdown)
@_check_poll_freshness
def node_can_shutdown(self, node_actor):
if not success:
if cancel_reason == self._node_shutdown.NODE_BROKEN:
self.cloud_nodes.blacklist(cloud_node_id)
- del self.shutdowns[cloud_node_id]
- del self.sizes_booting_shutdown[cloud_node_id]
elif cloud_node_id in self.booted:
self.booted.pop(cloud_node_id).actor.stop()
- del self.shutdowns[cloud_node_id]
- del self.sizes_booting_shutdown[cloud_node_id]
+ del self.shutdowns[cloud_node_id]
+ del self.sizes_booting_shutdown[cloud_node_id]
def shutdown(self):
self._logger.info("Shutting down after signal.")
poll_time = config.getint('Daemon', 'poll_time')
max_poll_time = config.getint('Daemon', 'max_poll_time')
- timer = TimedCallBackActor.start(poll_time / 10.0).proxy()
+ timer = TimedCallBackActor.start(poll_time / 10.0).tell_proxy()
cloud_node_poller = CloudNodeListMonitorActor.start(
- config.new_cloud_client(), timer, poll_time, max_poll_time).proxy()
+ config.new_cloud_client(), timer, poll_time, max_poll_time).tell_proxy()
arvados_node_poller = ArvadosNodeListMonitorActor.start(
- config.new_arvados_client(), timer, poll_time, max_poll_time).proxy()
+ config.new_arvados_client(), timer, poll_time, max_poll_time).tell_proxy()
job_queue_poller = JobQueueMonitorActor.start(
config.new_arvados_client(), timer, server_calculator,
- poll_time, max_poll_time).proxy()
+ poll_time, max_poll_time).tell_proxy()
return timer, cloud_node_poller, arvados_node_poller, job_queue_poller
_caught_signals = {}
server_calculator = build_server_calculator(config)
timer, cloud_node_poller, arvados_node_poller, job_queue_poller = \
launch_pollers(config, server_calculator)
- cloud_node_updater = node_update.start(config.new_cloud_client).proxy()
+ cloud_node_updater = node_update.start(config.new_cloud_client).tell_proxy()
node_daemon = NodeManagerDaemonActor.start(
job_queue_poller, arvados_node_poller, cloud_node_poller,
cloud_node_updater, timer,
config.getint('Daemon', 'boot_fail_after'),
config.getint('Daemon', 'node_stale_after'),
node_setup, node_shutdown, node_monitor,
- max_total_price=config.getfloat('Daemon', 'max_total_price')).proxy()
+ max_total_price=config.getfloat('Daemon', 'max_total_price')).tell_proxy()
signal.pause()
daemon_stopped = node_daemon.actor_ref.actor_stopped.is_set
"""
def __init__(self, max_sleep=1):
super(TimedCallBackActor, self).__init__()
- self._proxy = self.actor_ref.proxy()
+ self._proxy = self.actor_ref.tell_proxy()
self.messages = []
self.max_sleep = max_sleep
self.make_actor(2)
arv_node = testutil.arvados_node_mock(
2, hostname='compute-two.zzzzz.arvadosapi.com')
+ self.cloud_client.node_id.return_value = '2'
pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
self.assertEqual(self.cloud_mock.id, pair_id)
self.stop_proxy(self.node_actor)
cloud_size=get_cloud_size,
actor_ref=mock_actor)
mock_actor.proxy.return_value = mock_proxy
+ mock_actor.tell_proxy.return_value = mock_proxy
self.last_setup = mock_proxy
return mock_actor
self.cloud_factory().node_start_time.return_value = time.time()
self.cloud_updates = mock.MagicMock(name='updates_mock')
self.timer = testutil.MockTimer(deliver_immediately=False)
+ self.cloud_factory().node_id.side_effect = lambda node: node.id
self.node_setup = mock.MagicMock(name='setup_mock')
self.node_setup.start.side_effect = self.mock_node_start
def test_node_pairing_after_arvados_update(self):
cloud_node = testutil.cloud_node_mock(2)
self.make_daemon([cloud_node],
- [testutil.arvados_node_mock(2, ip_address=None)])
+ [testutil.arvados_node_mock(1, ip_address=None)])
arv_node = testutil.arvados_node_mock(2)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
--- /dev/null
+#!/usr/bin/env python
+
+from __future__ import absolute_import, print_function
+
+import errno
+import logging
+import threading
+import unittest
+
+import mock
+import pykka
+
+from . import testutil
+
+import arvnodeman.baseactor
+
+class BogusActor(arvnodeman.baseactor.BaseNodeManagerActor):
+ def __init__(self, e):
+ super(BogusActor, self).__init__()
+ self.exp = e
+
+ def doStuff(self):
+ raise self.exp
+
+class ActorUnhandledExceptionTest(unittest.TestCase):
+ def test_fatal_error(self):
+ for e in (MemoryError(), threading.ThreadError(), OSError(errno.ENOMEM, "")):
+ with mock.patch('os.killpg') as killpg_mock:
+ act = BogusActor.start(e).tell_proxy()
+ act.doStuff()
+ act.actor_ref.stop(block=True)
+ self.assertTrue(killpg_mock.called)
+
+ def test_nonfatal_error(self):
+ with mock.patch('os.killpg') as killpg_mock:
+ act = BogusActor.start(OSError(errno.ENOENT, "")).tell_proxy()
+ act.doStuff()
+ act.actor_ref.stop(block=True)
+ self.assertFalse(killpg_mock.called)
'job_uuid': job_uuid,
'crunch_worker_state': crunch_worker_state,
'properties': {},
- 'info': {'ping_secret': 'defaulttestsecret'}}
+ 'info': {'ping_secret': 'defaulttestsecret', 'ec2_instance_id': str(node_num)}}
node.update(kwargs)
return node
to_deliver = self.messages
self.messages = []
for callback, args, kwargs in to_deliver:
- callback(*args, **kwargs)
+ try:
+ callback(*args, **kwargs)
+ except pykka.ActorDeadError:
+ pass
def schedule(self, want_time, callback, *args, **kwargs):
with self.lock:
--- /dev/null
+#!/bin/sh
+
+set -e
+
+if ! test -d /sys/fs/cgroup ; then
+ echo "Arvbox requires cgroups to be mounted at /sys/fs/cgroup in order to use"
+ echo "Docker-in-Docker. Older operating systems that put cgroups in other"
+ echo "places (such as /cgroup) are not supported."
+ exit 1
+fi
+
+if ! which docker >/dev/null 2>/dev/null ; then
+ echo "Arvbox requires Docker. To install, run the following command as root:"
+ echo "curl -sSL https://get.docker.com/ | sh"
+ exit 1
+fi
+
+if test -z "$ARVBOX_DOCKER" ; then
+ if which greadlink >/dev/null 2>/dev/null ; then
+ ARVBOX_DOCKER=$(greadlink -f $(dirname $0)/../lib/arvbox/docker)
+ else
+ ARVBOX_DOCKER=$(readlink -f $(dirname $0)/../lib/arvbox/docker)
+ fi
+fi
+
+if test -z "$ARVBOX_CONTAINER" ; then
+ ARVBOX_CONTAINER=arvbox
+fi
+
+if test -z "$ARVBOX_BASE" ; then
+ ARVBOX_BASE="$HOME/.arvbox"
+fi
+
+if test -z "$ARVBOX_DATA" ; then
+ ARVBOX_DATA="$ARVBOX_BASE/$ARVBOX_CONTAINER"
+fi
+
+if test -z "$ARVADOS_ROOT" ; then
+ ARVADOS_ROOT="$ARVBOX_DATA/arvados"
+fi
+
+if test -z "$ARVADOS_DEV_ROOT" ; then
+ ARVADOS_DEV_ROOT="$ARVBOX_DATA/arvados-dev"
+fi
+
+if test -z "$SSO_ROOT" ; then
+ SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider"
+fi
+
+PG_DATA="$ARVBOX_DATA/postgres"
+VAR_DATA="$ARVBOX_DATA/var"
+PASSENGER="$ARVBOX_DATA/passenger"
+GEMS="$ARVBOX_DATA/gems"
+
+getip() {
+ docker inspect $ARVBOX_CONTAINER | grep \"IPAddress\" | head -n1 | tr -d ' ":,\n' | cut -c10-
+}
+
+gethost() {
+ set +e
+ OVERRIDE=$(docker exec -i $ARVBOX_CONTAINER cat /var/run/localip_override 2>/dev/null)
+ CODE=$?
+ set -e
+ if test "$CODE" = 0 ; then
+ echo $OVERRIDE
+ else
+ getip
+ fi
+}
+
+updateconf() {
+ if test -f ~/.config/arvados/$ARVBOX_CONTAINER.conf ; then
+ sed "s/ARVADOS_API_HOST=.*/ARVADOS_API_HOST=$(gethost):8000/" <$HOME/.config/arvados/$ARVBOX_CONTAINER.conf >$HOME/.config/arvados/$ARVBOX_CONTAINER.conf.tmp
+ mv ~/.config/arvados/$ARVBOX_CONTAINER.conf.tmp ~/.config/arvados/$ARVBOX_CONTAINER.conf
+ else
+ mkdir -p $HOME/.config/arvados
+ cat >$HOME/.config/arvados/$ARVBOX_CONTAINER.conf <<EOF
+ARVADOS_API_HOST=$(gethost):8000
+ARVADOS_API_TOKEN=
+ARVADOS_API_HOST_INSECURE=true
+EOF
+ fi
+}
+
+wait_for_arvbox() {
+ FF=/tmp/arvbox-fifo-$$
+ mkfifo $FF
+ docker logs -f $ARVBOX_CONTAINER > $FF &
+ LOGPID=$!
+ while read line ; do
+ echo $line
+ if echo $line | grep "Workbench is running at" >/dev/null ; then
+ kill $LOGPID
+ fi
+ done < $FF
+ rm $FF
+ echo
+ if test -n "$localip" ; then
+ echo "export ARVADOS_API_HOST=$localip:8000"
+ else
+ echo "export ARVADOS_API_HOST=$(gethost):8000"
+ fi
+}
+
+run() {
+ if docker ps -a | grep -E "$ARVBOX_CONTAINER$" -q ; then
+ echo "Container $ARVBOX_CONTAINER is already running, use stop, restart or rebuild"
+ exit 1
+ fi
+
+ if echo "$1" | grep '^public' ; then
+ if test -n "$ARVBOX_PUBLISH_IP" ; then
+ localip=$ARVBOX_PUBLISH_IP
+ else
+ defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
+ localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
+ fi
+ iptemp=$(tempfile)
+ echo $localip > $iptemp
+ chmod og+r $iptemp
+ PUBLIC="--volume=$iptemp:/var/run/localip_override
+ --publish=80:80
+ --publish=8000:8000
+ --publish=8900:8900
+ --publish=9001:9001
+ --publish=9002:9002
+ --publish=25100:25100
+ --publish=25107:25107
+ --publish=25108:25108
+ --publish=8001:8001"
+ else
+ PUBLIC=""
+ fi
+
+ if echo "$1" | grep 'demo$' ; then
+ if test -d "$ARVBOX_DATA" ; then
+ echo "It looks like you already have a development container named $ARVBOX_CONTAINER."
+ echo "Set ARVBOX_CONTAINER to set a different name for your demo container"
+ exit 1
+ fi
+
+ if ! (docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q) ; then
+ docker create -v /var/lib/postgresql -v /var/lib/arvados --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true
+ fi
+
+ docker run \
+ --detach \
+ --name=$ARVBOX_CONTAINER \
+ --privileged \
+ --volumes-from $ARVBOX_CONTAINER-data \
+ $PUBLIC \
+ arvados/arvbox-demo
+ updateconf
+ wait_for_arvbox
+ else
+ mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS"
+
+ if ! test -d "$ARVADOS_ROOT" ; then
+ git clone https://github.com/curoverse/arvados.git "$ARVADOS_ROOT"
+ fi
+ if ! test -d "$SSO_ROOT" ; then
+ git clone https://github.com/curoverse/sso-devise-omniauth-provider.git "$SSO_ROOT"
+ fi
+
+ if test "$1" = test ; then
+ shift
+
+ if ! test -d "$ARVADOS_DEV_ROOT" ; then
+ git clone https://github.com/curoverse/arvados-dev.git "$ARVADOS_DEV_ROOT"
+ fi
+
+ mkdir -p $VAR_DATA/test
+
+ docker run \
+ --detach \
+ --name=$ARVBOX_CONTAINER \
+ --privileged \
+ "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
+ "--volume=$ARVADOS_DEV_ROOT:/usr/src/arvados-dev:rw" \
+ "--volume=$SSO_ROOT:/usr/src/sso:rw" \
+ "--volume=$PG_DATA:/var/lib/postgresql:rw" \
+ "--volume=$VAR_DATA:/var/lib/arvados:rw" \
+ "--volume=$PASSENGER:/var/lib/passenger:rw" \
+ "--volume=$GEMS:/var/lib/gems:rw" \
+ arvados/arvbox-dev \
+ /usr/local/bin/runsvinit -svdir=/etc/test-service
+
+ docker exec -ti \
+ $ARVBOX_CONTAINER \
+ /usr/local/lib/arvbox/runsu.sh \
+ /usr/local/lib/arvbox/waitforpostgres.sh
+
+ docker exec -ti \
+ $ARVBOX_CONTAINER \
+ /usr/local/lib/arvbox/runsu.sh \
+ /var/lib/arvbox/service/sso/run-service --only-setup
+
+ docker exec -ti \
+ $ARVBOX_CONTAINER \
+ /usr/local/lib/arvbox/runsu.sh \
+ /var/lib/arvbox/service/api/run-service --only-setup
+
+ docker exec -ti \
+ $ARVBOX_CONTAINER \
+ /usr/local/lib/arvbox/runsu.sh \
+ /usr/src/arvados-dev/jenkins/run-tests.sh \
+ --temp /var/lib/arvados/test \
+ WORKSPACE=/usr/src/arvados \
+ GEM_HOME=/var/lib/gems \
+ "$@"
+ elif echo "$1" | grep 'dev$' ; then
+ docker run \
+ --detach \
+ --name=$ARVBOX_CONTAINER \
+ --privileged \
+ "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
+ "--volume=$SSO_ROOT:/usr/src/sso:rw" \
+ "--volume=$PG_DATA:/var/lib/postgresql:rw" \
+ "--volume=$VAR_DATA:/var/lib/arvados:rw" \
+ "--volume=$PASSENGER:/var/lib/passenger:rw" \
+ "--volume=$GEMS:/var/lib/gems:rw" \
+ $PUBLIC \
+ arvados/arvbox-dev
+ updateconf
+ wait_for_arvbox
+ echo "The Arvados source code is checked out at: $ARVADOS_ROOT"
+ else
+ echo "Unknown configuration '$1'"
+ fi
+ fi
+}
+
+stop() {
+ if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
+ docker stop $ARVBOX_CONTAINER
+ fi
+
+ VOLUMES=--volumes=true
+ if docker ps -a --filter "status=created" | grep -E "$ARVBOX_CONTAINER$" -q ; then
+ docker rm $VOLUMES $ARVBOX_CONTAINER
+ fi
+ if docker ps -a --filter "status=exited" | grep -E "$ARVBOX_CONTAINER$" -q ; then
+ docker rm $VOLUMES $ARVBOX_CONTAINER
+ fi
+}
+
+build() {
+ if ! test -f "$ARVBOX_DOCKER/Dockerfile.base" ; then
+ echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
+ exit 1
+ fi
+ docker build -t arvados/arvbox-base -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
+ if test "$1" = localdemo -o "$1" = publicdemo ; then
+ docker build -t arvados/arvbox-demo -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+ else
+ docker build -t arvados/arvbox-dev -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+ fi
+}
+
+check() {
+ case "$1" in
+ localdemo|publicdemo|dev|publicdev|test)
+ true
+ ;;
+ *)
+ echo "Argument to $subcmd must be one of localdemo, publicdemo, dev, publicdev, test"
+ exit 1
+ ;;
+ esac
+}
+
+subcmd="$1"
+if test -n "$subcmd" ; then
+ shift
+fi
+case "$subcmd" in
+ build)
+ check $@
+ build $@
+ ;;
+
+ start|run)
+ check $@
+ run $@
+ ;;
+
+ sh*)
+ exec docker exec -ti $ARVBOX_CONTAINER /usr/bin/env TERM=$TERM GEM_HOME=/var/lib/gems /bin/bash
+ ;;
+
+ pipe)
+ exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash -
+ ;;
+
+ stop)
+ stop
+ ;;
+
+ restart)
+ check $@
+ stop
+ run $@
+ ;;
+
+ rebuild)
+ check $@
+ stop
+ build $@
+ run $@
+ ;;
+
+ ip)
+ getip
+ ;;
+
+ host)
+ gethost
+ ;;
+
+ open)
+ exec xdg-open http://$(gethost)
+ ;;
+
+ status)
+ echo "Selected: $ARVBOX_CONTAINER"
+ if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
+ echo "Status: running"
+ echo "Container IP: $(getip)"
+ echo "Published host: $(gethost)"
+ else
+ echo "Status: not running"
+ fi
+ if test -d "$ARVBOX_DATA" ; then
+ echo "Data: $ARVBOX_DATA"
+ elif docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q ; then
+ echo "Data: $ARVBOX_CONTAINER-data"
+ else
+ echo "Data: none"
+ fi
+ ;;
+
+ reset|destroy)
+ stop
+ if test -d "$ARVBOX_DATA" ; then
+ if test "$subcmd" = destroy ; then
+ if test "$1" != -f ; then
+ echo "WARNING! This will delete your entire arvbox ($ARVBOX_DATA)."
+ echo "Use destroy -f if you really mean it."
+ exit 1
+ fi
+ set -x
+ rm -rf "$ARVBOX_DATA"
+ else
+ if test "$1" != -f ; then
+ echo "WARNING! This will delete your arvbox data ($ARVBOX_DATA)."
+ echo "Code and downloaded packages will be preserved."
+ echo "Use reset -f if you really mean it."
+ exit 1
+ fi
+ set -x
+ rm -rf "$ARVBOX_DATA/postgres"
+ rm -rf "$ARVBOX_DATA/var"
+ fi
+ else
+ if test "$1" != -f ; then
+ echo "WARNING! This will delete your data container $ARVBOX_CONTAINER-data. Use -f if you really mean it."
+ exit 1
+ fi
+ set -x
+ docker rm "$ARVBOX_CONTAINER-data"
+ fi
+ ;;
+
+ log)
+ if test -n "$1" ; then
+ exec docker exec -ti $ARVBOX_CONTAINER /usr/bin/env TERM=$TERM less --follow-name +GF "/etc/service/$1/log/main/current"
+ else
+ exec docker exec -ti $ARVBOX_CONTAINER /usr/bin/env TERM=$TERM tail $(docker exec -ti $ARVBOX_CONTAINER find -L /etc -path '/etc/service/*/log/main/current' -printf " %p")
+ fi
+ ;;
+
+ cat)
+ if test -n "$1" ; then
+ exec docker exec -ti $ARVBOX_CONTAINER cat "$@"
+ else
+ echo "Usage: $0 $subcmd <files>"
+ fi
+ ;;
+
+ ls)
+ exec docker exec -ti $ARVBOX_CONTAINER /usr/bin/env TERM=$TERM ls "$@"
+ ;;
+
+ sv)
+ if test -n "$1" -a -n "$2" ; then
+ exec docker exec -ti $ARVBOX_CONTAINER sv "$@"
+ else
+ echo "Usage: $0 $subcmd <start|stop|restart> <service>"
+ echo "Available services:"
+ exec docker exec -ti $ARVBOX_CONTAINER ls /etc/service
+ fi
+ ;;
+
+ clone)
+ if test -n "$2" ; then
+ cp -r "$ARVBOX_BASE/$1" "$ARVBOX_BASE/$2"
+ echo "Created new arvbox $2"
+ echo "export ARVBOX_CONTAINER=$2"
+ else
+ echo "clone <from> <to> clone an arvbox"
+ echo "available arvboxes: $(ls $ARVBOX_BASE)"
+ fi
+ ;;
+
+ *)
+ echo "Arvados-in-a-box http://arvados.org"
+ echo
+ echo "$(basename $0) (build|start|run|open|shell|ip|stop|rebuild|reset|destroy|log|svrestart)"
+ echo
+ echo "build <config> build arvbox Docker image"
+ echo "start|run <config> start $ARVBOX_CONTAINER container"
+ echo "open open arvbox workbench in a web browser"
+ echo "shell enter arvbox shell"
+ echo "ip print arvbox docker container ip address"
+ echo "host print arvbox published host"
+ echo "status print some information about current arvbox"
+ echo "stop stop arvbox container"
+ echo "restart <config> stop, then run again"
+ echo "rebuild <config> stop, build arvbox Docker image, run"
+ echo "reset delete arvbox arvados data (be careful!)"
+ echo "destroy delete all arvbox code and data (be careful!)"
+ echo "log <service> tail log of specified service"
+ echo "ls <options> list directories inside arvbox"
+ echo "cat <files> get contents of files inside arvbox"
+ echo "pipe run a bash script piped in from stdin"
+ echo "sv <start|stop|restart> <service> change state of service inside arvbox"
+ echo "clone <from> <to> clone an arvbox"
+ ;;
+esac
--- /dev/null
+FROM debian:8
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -yq install \
+ postgresql-9.4 git gcc golang-go runit \
+ ruby rake bundler curl libpq-dev \
+ libcurl4-openssl-dev libssl-dev zlib1g-dev libpcre3-dev \
+ openssh-server python-setuptools netcat-traditional \
+ libpython-dev fuse libfuse-dev python-pip python-yaml \
+ pkg-config libattr1-dev python-llfuse python-pycurl \
+ libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
+ libjson-perl nginx gitolite3 lsof python-epydoc graphviz \
+ apt-transport-https ca-certificates slurm-wlm
+
+VOLUME /var/lib/docker
+VOLUME /var/log/nginx
+VOLUME /etc/ssl/private
+
+RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \
+ apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+
+RUN mkdir -p /etc/apt/sources.list.d && \
+ echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \
+ apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -yq install docker-engine=1.9.1-0~jessie
+
+RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql
+
+RUN cd /root && \
+ GOPATH=$PWD go get github.com/curoverse/runsvinit && \
+ install bin/runsvinit /usr/local/bin
+
+ADD fuse.conf /etc/
+
+ADD crunch-setup.sh gitolite.rc \
+ keep-setup.sh common.sh createusers.sh \
+ logger runsu.sh waitforpostgres.sh \
+ application_yml_override.py \
+ /usr/local/lib/arvbox/
+
+# Start the supervisor.
+CMD ["/usr/local/bin/runsvinit"]
--- /dev/null
+FROM arvados/arvbox-base
+
+RUN cd /usr/src && \
+ git clone https://github.com/curoverse/arvados.git && \
+ git clone https://github.com/curoverse/sso-devise-omniauth-provider.git sso
+
+ADD service/ /var/lib/arvbox/service
+RUN rmdir /etc/service && ln -sf /var/lib/arvbox/service /etc
+
+RUN chown -R 1000:1000 /usr/src && /usr/local/lib/arvbox/createusers.sh
+
+RUN sudo -u arvbox /var/lib/arvbox/service/sso/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/workbench/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
--- /dev/null
+FROM arvados/arvbox-base
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -yq install \
+ python-virtualenv python3-virtualenv linkchecker xvfb iceweasel
+
+RUN set -e && \
+ PJS=phantomjs-1.9.7-linux-x86_64 && \
+ curl -L -o/tmp/$PJS.tar.bz2 https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 && \
+ tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
+ ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+
+ADD service/ /var/lib/arvbox/service
+RUN rmdir /etc/service && ln -sf /var/lib/arvbox/service /etc
+
+RUN mkdir /etc/test-service && ln -sf /var/lib/arvbox/service/postgres /etc/test-service
--- /dev/null
+#!/usr/bin/env python
+
+import yaml
+
+try:
+ with open("application.yml.override") as f:
+ b = yaml.load(f)
+except IOError:
+ exit()
+
+with open("application.yml") as f:
+ a = yaml.load(f)
+
+def recursiveMerge(a, b):
+ if isinstance(a, dict) and isinstance(b, dict):
+ for k in b:
+ print k
+ a[k] = recursiveMerge(a.get(k), b[k])
+ return a
+ else:
+ return b
+
+with open("application.yml", "w") as f:
+ yaml.dump(recursiveMerge(a, b), f)
--- /dev/null
+
+if test -s /var/run/localip_override ; then
+ localip=$(cat /var/run/localip_override)
+else
+ defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
+ localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
+fi
+
+export GEM_HOME=/var/lib/gems
+export GEM_PATH=/var/lib/gems
+
+declare -A services
+services=(
+ [workbench]=80
+ [api]=8000
+ [sso]=8900
+ [arv-git-httpd]=9001
+ [keep-web]=9002
+ [keepproxy]=25100
+ [keepstore0]=25107
+ [keepstore1]=25108
+ [ssh]=22
+ [doc]=8001
+)
+
+if test "$(id arvbox -u 2>/dev/null)" = 0 ; then
+ PGUSER=postgres
+ PGGROUP=postgres
+else
+ PGUSER=arvbox
+ PGGROUP=arvbox
+fi
+
+run_bundler() {
+ if test -f Gemfile.lock ; then
+ frozen=--frozen
+ else
+ frozen=""
+ fi
+ if ! flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
+ flock /var/lib/arvados/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
+ fi
+}
+
+pip_install() {
+ pushd /var/lib/arvados/pip
+ for p in $(ls http*.tar.gz) ; do
+ if test -f $p ; then
+ ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
+ fi
+ done
+ for p in $(ls http*.whl) ; do
+ if test -f $p ; then
+ ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
+ fi
+ done
+ popd
+
+ if ! pip install --no-index --find-links /var/lib/arvados/pip $1 ; then
+ pip install $1
+ fi
+}
--- /dev/null
+#!/bin/bash
+
+set -e -o pipefail
+
+if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
+ HOSTUID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f4)
+ HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5)
+ FUSEGID=$(ls -nd /dev/fuse | sed 's/ */ /' | cut -d' ' -f5)
+
+ mkdir -p /var/lib/arvados/git /var/lib/gems /var/lib/passenger
+
+ groupadd --gid $HOSTGID --non-unique arvbox
+ groupadd --gid $FUSEGID --non-unique fuse
+ groupadd --gid $HOSTGID --non-unique git
+ useradd --home-dir /var/lib/arvados \
+ --uid $HOSTUID --gid $HOSTGID \
+ --non-unique \
+ --groups docker,fuse \
+ arvbox
+ useradd --home-dir /var/lib/arvados/git --uid $HOSTUID --gid $HOSTGID --non-unique git
+ useradd --groups docker,fuse crunch
+
+ chown arvbox:arvbox -R /usr/local /var/lib/arvados /var/lib/gems \
+ /var/lib/passenger /var/lib/postgresql \
+ /var/lib/nginx /var/log/nginx /etc/ssl/private
+
+ mkdir -p /var/lib/gems/ruby/2.1.0
+ chown arvbox:arvbox -R /var/lib/gems/ruby/2.1.0
+
+ mkdir -p /tmp/crunch0 /tmp/crunch1
+ chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
+
+ echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
+fi
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
+install bin/crunchstat /usr/local/bin
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /usr/src/arvados/services/api/superuser_token)
+export CRUNCH_JOB_BIN=/usr/src/arvados/sdk/cli/bin/crunch-job
+export PERLLIB=/usr/src/arvados/sdk/perl/lib
+export CRUNCH_TMP=/tmp/$1
+export CRUNCH_DISPATCH_LOCKFILE=/var/lock/$1-dispatch
+export CRUNCH_JOB_DOCKER_BIN=docker
+export HOME=/tmp/$1
+
+cd /usr/src/arvados/services/api
+exec bundle exec ./script/crunch-dispatch.rb development
--- /dev/null
+user_allow_other
--- /dev/null
+# This is based on the default Gitolite configuration file with the following
+# changes applied as described here:
+# http://doc.arvados.org/install/install-arv-git-httpd.html
+
+# configuration variables for gitolite
+
+# This file is in perl syntax. But you do NOT need to know perl to edit it --
+# just mind the commas, use single quotes unless you know what you're doing,
+# and make sure the brackets and braces stay matched up!
+
+# (Tip: perl allows a comma after the last item in a list also!)
+
+# HELP for commands can be had by running the command with "-h".
+
+# HELP for all the other FEATURES can be found in the documentation (look for
+# "list of non-core programs shipped with gitolite" in the master index) or
+# directly in the corresponding source file.
+
+my $repo_aliases;
+my $aliases_src = "$ENV{HOME}/.gitolite/arvadosaliases.pl";
+if ($ENV{HOME} && (-e $aliases_src)) {
+ $repo_aliases = do $aliases_src;
+}
+$repo_aliases ||= {};
+
+%RC = (
+
+ REPO_ALIASES => $repo_aliases,
+
+ # ------------------------------------------------------------------
+
+ # default umask gives you perms of '0700'; see the rc file docs for
+ # how/why you might change this
+ UMASK => 0022,
+
+ # look for "git-config" in the documentation
+ GIT_CONFIG_KEYS => '',
+
+ # comment out if you don't need all the extra detail in the logfile
+ LOG_EXTRA => 1,
+ # logging options
+ # 1. leave this section as is for 'normal' gitolite logging (default)
+ # 2. uncomment this line to log ONLY to syslog:
+ # LOG_DEST => 'syslog',
+ # 3. uncomment this line to log to syslog and the normal gitolite log:
+ # LOG_DEST => 'syslog,normal',
+ # 4. prefixing "repo-log," to any of the above will **also** log just the
+ # update records to "gl-log" in the bare repo directory:
+ # LOG_DEST => 'repo-log,normal',
+ # LOG_DEST => 'repo-log,syslog',
+ # LOG_DEST => 'repo-log,syslog,normal',
+
+ # roles. add more roles (like MANAGER, TESTER, ...) here.
+ # WARNING: if you make changes to this hash, you MUST run 'gitolite
+ # compile' afterward, and possibly also 'gitolite trigger POST_COMPILE'
+ ROLES => {
+ READERS => 1,
+ WRITERS => 1,
+ },
+
+ # enable caching (currently only Redis). PLEASE RTFM BEFORE USING!!!
+ # CACHE => 'Redis',
+
+ # ------------------------------------------------------------------
+
+ # rc variables used by various features
+
+ # the 'info' command prints this as additional info, if it is set
+ # SITE_INFO => 'Please see http://blahblah/gitolite for more help',
+
+ # the CpuTime feature uses these
+ # display user, system, and elapsed times to user after each git operation
+ # DISPLAY_CPU_TIME => 1,
+ # display a warning if total CPU times (u, s, cu, cs) crosses this limit
+ # CPU_TIME_WARN_LIMIT => 0.1,
+
+ # the Mirroring feature needs this
+ # HOSTNAME => "foo",
+
+ # TTL for redis cache; PLEASE SEE DOCUMENTATION BEFORE UNCOMMENTING!
+ # CACHE_TTL => 600,
+
+ # ------------------------------------------------------------------
+
+ # suggested locations for site-local gitolite code (see cust.html)
+
+ # this one is managed directly on the server
+ # LOCAL_CODE => "$ENV{HOME}/local",
+
+ # or you can use this, which lets you put everything in a subdirectory
+ # called "local" in your gitolite-admin repo. For a SECURITY WARNING
+ # on this, see http://gitolite.com/gitolite/non-core.html#pushcode
+ # LOCAL_CODE => "$rc{GL_ADMIN_BASE}/local",
+
+ # ------------------------------------------------------------------
+
+ # List of commands and features to enable
+
+ ENABLE => [
+
+ # COMMANDS
+
+ # These are the commands enabled by default
+ 'help',
+ 'desc',
+ 'info',
+ 'perms',
+ 'writable',
+
+ # Uncomment or add new commands here.
+ # 'create',
+ # 'fork',
+ # 'mirror',
+ # 'readme',
+ # 'sskm',
+ # 'D',
+
+ # These FEATURES are enabled by default.
+
+ # essential (unless you're using smart-http mode)
+ 'ssh-authkeys',
+
+ # creates git-config enties from gitolite.conf file entries like 'config foo.bar = baz'
+ 'git-config',
+
+ # creates git-daemon-export-ok files; if you don't use git-daemon, comment this out
+ 'daemon',
+
+ # creates projects.list file; if you don't use gitweb, comment this out
+ 'gitweb',
+
+ # These FEATURES are disabled by default; uncomment to enable. If you
+ # need to add new ones, ask on the mailing list :-)
+
+ # user-visible behaviour
+
+ # prevent wild repos auto-create on fetch/clone
+ # 'no-create-on-read',
+ # no auto-create at all (don't forget to enable the 'create' command!)
+ # 'no-auto-create',
+
+ # access a repo by another (possibly legacy) name
+ 'Alias',
+
+ # give some users direct shell access. See documentation in
+ # sts.html for details on the following two choices.
+ # "Shell $ENV{HOME}/.gitolite.shell-users",
+ # 'Shell alice bob',
+
+ # set default roles from lines like 'option default.roles-1 = ...', etc.
+ # 'set-default-roles',
+
+ # show more detailed messages on deny
+ # 'expand-deny-messages',
+
+ # show a message of the day
+ # 'Motd',
+
+ # system admin stuff
+
+ # enable mirroring (don't forget to set the HOSTNAME too!)
+ # 'Mirroring',
+
+ # allow people to submit pub files with more than one key in them
+ # 'ssh-authkeys-split',
+
+ # selective read control hack
+ # 'partial-copy',
+
+ # manage local, gitolite-controlled, copies of read-only upstream repos
+ # 'upstream',
+
+ # updates 'description' file instead of 'gitweb.description' config item
+ # 'cgit',
+
+ # allow repo-specific hooks to be added
+ # 'repo-specific-hooks',
+
+ # performance, logging, monitoring...
+
+ # be nice
+ # 'renice 10',
+
+ # log CPU times (user, system, cumulative user, cumulative system)
+ # 'CpuTime',
+
+ # syntactic_sugar for gitolite.conf and included files
+
+ # allow backslash-escaped continuation lines in gitolite.conf
+ # 'continuation-lines',
+
+ # create implicit user groups from directory names in keydir/
+ # 'keysubdirs-as-groups',
+
+ # allow simple line-oriented macros
+ # 'macros',
+
+ # Kindergarten mode
+
+ # disallow various things that sensible people shouldn't be doing anyway
+ # 'Kindergarten',
+ ],
+
+);
+
+# ------------------------------------------------------------------------------
+# per perl rules, this should be the last line in such a file:
+1;
+
+# Local variables:
+# mode: perl
+# End:
+# vim: set syn=perl:
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+sleep 2
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepstore"
+install bin/keepstore /usr/local/bin
+
+mkdir -p /var/lib/arvados/$1
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+set +e
+read -rd $'\000' keepservice <<EOF
+{
+ "service_host":"$localip",
+ "service_port":$2,
+ "service_ssl_flag":false,
+ "service_type":"disk"
+}
+EOF
+set -e
+
+if test -s /var/lib/arvados/$1-uuid ; then
+ keep_uuid=$(cat /var/lib/arvados/$1-uuid)
+ arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
+else
+ UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
+ echo $UUID > /var/lib/arvados/$1-uuid
+fi
+
+set +e
+killall -HUP keepproxy
+
+exec /usr/local/bin/keepstore \
+ -listen=:$2 \
+ -enforce-permissions=true \
+ -blob-signing-key-file=/var/lib/arvados/blob_signing_key \
+ -max-buffers=20 \
+ -volume=/var/lib/arvados/$1
--- /dev/null
+#!/bin/sh
+exec svlogd -tt ./main
--- /dev/null
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
--- /dev/null
+Copyright (c) 2015, Kosma Moczek
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of runit-docker nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
--- /dev/null
+CFLAGS=-std=c99 -Wall -O2 -fPIC -D_POSIX_SOURCE -D_GNU_SOURCE
+LDLIBS=-ldl
+
+PROGNAME=runit-docker
+
+all: $(PROGNAME).so
+
+%.so: %.c
+ gcc -shared $(CFLAGS) $(LDLIBS) -o $@ $^
+
+install: runit-docker.so
+ mkdir -p $(DESTDIR)/sbin
+ mkdir -p $(DESTDIR)/lib
+ install -m 755 $(PROGNAME) $(DESTDIR)/sbin/
+ install -m 755 $(PROGNAME).so $(DESTDIR)/lib/
+
+clean:
+ $(RM) $(PROGNAME).so
--- /dev/null
+# runit-docker
+
+Docker and `runsvdir` don't quite agree on what each signal means, causing
+TONS of frustration when attempting to use `runsvdir` as init under Docker.
+`runit-docker` is a plug'n'play adapter library which does signal translation
+without the overhead and nuisance of running a nanny process.
+
+## Features
+
+* Pressing Ctrl-C does a clean shutdown.
+* `docker stop` does a clean shutdown.
+
+Under the hood, `runit-docker` translates `SIGTERM` and `SIGINT` to `SIGHUP`.
+
+## Usage
+
+* Build with `make`, install with `make install`.
+* Add `CMD ["/sbin/runit-docker"]` to your `Dockerfile`.
+* Run `debian/rules clean build binary` to build a Debian package.
+
+## Author
+
+runit-docker was written by Kosma Moczek <kosma.moczek@pixers.pl> during a single Scrum
+planning meeting. Damn meetings.
--- /dev/null
+runit-docker (1.1) unstable; urgency=low
+
+ * Simplify logic.
+ * Install for SIGINT as well.
+
+ -- Kosma Moczek <kosma@kosma.pl> Mon, 11 May 2015 12:23:59 +0000
+
+runit-docker (1.0) unstable; urgency=low
+
+ * Initial release
+
+ -- Kosma Moczek <kosma@kosma.pl> Mon, 11 May 2015 12:23:59 +0000
--- /dev/null
+Source: runit-docker
+Section: contrib/admin
+Priority: optional
+Maintainer: Kosma Moczek <kosma@kosma.pl>
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.5
+Homepage: https://github.com/kosma/runit-docker
+#Vcs-Git: git://anonscm.debian.org/collab-maint/runit-docker.git
+#Vcs-Browser: http://anonscm.debian.org/?p=collab-maint/runit-docker.git;a=summary
+
+Package: runit-docker
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: painlessly use runit in Docker containers
--- /dev/null
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: runit-docker
+Source: https://github.com/kosma/runit-docker
+
+Files: *
+Copyright: 2015 Kosma Moczek <kosma@kosma.pl>
+License: MIT
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name of runit-docker nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- /dev/null
+#!/usr/bin/make -f
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#DH_VERBOSE = 1
+
+# see EXAMPLES in dpkg-buildflags(1) and read /usr/share/dpkg/*
+DPKG_EXPORT_BUILDFLAGS = 1
+include /usr/share/dpkg/default.mk
+
+# see FEATURE AREAS in dpkg-buildflags(1)
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+# see ENVIRONMENT in dpkg-buildflags(1)
+# package maintainers to append CFLAGS
+#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic
+# package maintainers to append LDFLAGS
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+
+# main packaging script based on dh7 syntax
+%:
+ dh $@
+
+# debmake generated override targets
+# This is example for Cmake (See http://bugs.debian.org/641051 )
+#override_dh_auto_configure:
+# dh_auto_configure -- \
+# -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH)
+
+
+
+
--- /dev/null
+3.0 (quilt)
--- /dev/null
+#!/bin/sh
+
+export LD_PRELOAD=/lib/runit-docker.so
+exec runsvdir /etc/service
--- /dev/null
+#include <signal.h>
+#include <dlfcn.h>
+#include <stdlib.h>
+
+
+int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact)
+{
+ static int (*real_sigaction)(int signum, const struct sigaction *act, struct sigaction *oldact) = NULL;
+
+ // Retrieve the real sigaction we just shadowed.
+ if (real_sigaction == NULL) {
+ real_sigaction = (void *) dlsym(RTLD_NEXT, "sigaction");
+ // Prevent further shadowing in children.
+ unsetenv("LD_PRELOAD");
+ }
+
+ if (signum == SIGTERM) {
+ // Skip this handler, it doesn't do what we want.
+ return 0;
+ }
+
+ if (signum == SIGHUP) {
+ // Install this handler for others as well.
+ real_sigaction(SIGTERM, act, oldact);
+ real_sigaction(SIGINT, act, oldact);
+ }
+
+ // Forward the call the the real sigaction.
+ return real_sigaction(signum, act, oldact);
+}
+
+// vim: ts=2 sw=2 et
--- /dev/null
+#!/bin/sh
+
+HOSTUID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f4)
+HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5)
+
+flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh
+
+export HOME=/var/lib/arvados
+
+if test -z "$1" ; then
+ exec chpst -u arvbox:arvbox:docker $0-service
+else
+ exec chpst -u arvbox:arvbox:docker $@
+fi
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/arvados/services/api
+export RAILS_ENV=development
+
+run_bundler --without=development
+bundle exec passenger start --runtime-check-only --runtime-dir=/var/lib/passenger
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+set -u
+
+if ! test -s /var/lib/arvados/api_uuid_prefix ; then
+ ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"' > /var/lib/arvados/api_uuid_prefix
+fi
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+
+if ! test -s /var/lib/arvados/api_secret_token ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/api_secret_token
+fi
+secret_token=$(cat /var/lib/arvados/api_secret_token)
+
+if ! test -s /var/lib/arvados/blob_signing_key ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/blob_signing_key
+fi
+blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+
+# self signed key will be created by SSO server script.
+test -s /var/lib/arvados/self-signed.key
+
+sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
+
+if test -s /var/lib/arvados/vm-uuid ; then
+ vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+else
+ vm_uuid=$uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]')
+ echo $vm_uuid > /var/lib/arvados/vm-uuid
+fi
+
+cat >config/application.yml <<EOF
+development:
+ uuid_prefix: $uuid_prefix
+ secret_token: $secret_token
+ blob_signing_key: $blob_signing_key
+ sso_app_secret: $sso_app_secret
+ sso_app_id: arvados-server
+ sso_provider_url: "https://$localip:${services[sso]}"
+ sso_insecure: true
+ workbench_address: "http://$localip/"
+ git_repo_ssh_base: "git@$localip:"
+ git_repo_https_base: "http://$localip:${services[arv-git-httpd]}/"
+ new_users_are_active: true
+ auto_admin_first_user: true
+ auto_setup_new_users: true
+ auto_setup_new_users_with_vm_uuid: $vm_uuid
+ auto_setup_new_users_with_repository: true
+ default_collection_replication: 1
+EOF
+
+(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+
+if ! test -f /var/lib/arvados/api_database_pw ; then
+ ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
+fi
+database_pw=$(cat /var/lib/arvados/api_database_pw)
+
+if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
+ psql postgres -c "create user arvados with password '$database_pw'"
+ psql postgres -c "ALTER USER arvados CREATEDB;"
+fi
+
+sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config/database.yml
+
+if ! test -f /var/lib/arvados/api_database_setup ; then
+ bundle exec rake db:setup
+ touch /var/lib/arvados/api_database_setup
+fi
+
+if ! test -s /var/lib/arvados/superuser_token ; then
+ bundle exec ./script/create_superuser_token.rb > /var/lib/arvados/superuser_token
+fi
+
+rm -rf tmp
+
+bundle exec rake db:migrate
+
+set +u
+if test "$1" = "--only-setup" ; then
+ exit
+fi
+
+ARVADOS_WEBSOCKETS=1 exec bundle exec passenger start --port=${services[api]} \
+ --runtime-dir=/var/lib/passenger \
+ --ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \
+ --ssl-certificate-key=/var/lib/arvados/self-signed.key
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/arv-git-httpd"
+install bin/arv-git-httpd /usr/local/bin
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export GITOLITE_HTTP_HOME=/var/lib/arvados/git
+export GL_BYPASS_ACCESS_CHECKS=1
+export PATH="$PATH:/var/lib/arvados/git/bin"
+cd ~git
+
+exec /usr/local/bin/arv-git-httpd \
+ -address=:${services[arv-git-httpd]} \
+ -git-command=/usr/share/gitolite3/gitolite-shell \
+ -repo-root=/var/lib/arvados/git/repositories
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-run"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunch-dispatch-local"
+install bin/crunch-run bin/crunch-dispatch-local /usr/local/bin
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+exec /usr/local/lib/arvbox/crunch-setup.sh crunch0
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+sleep 1
+exec /usr/local/lib/arvbox/crunch-setup.sh crunch1
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/arvados/doc
+run_bundler --without=development
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+set -u
+
+cat <<EOF >/var/lib/arvados/doc-nginx.conf
+worker_processes auto;
+pid /var/lib/arvados/doc-nginx.pid;
+
+error_log stderr;
+daemon off;
+
+events {
+ worker_connections 64;
+}
+
+http {
+ access_log off;
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+ server {
+ listen ${services[doc]} default_server;
+ listen [::]:${services[doc]} default_server;
+ root /usr/src/arvados/doc/.site;
+ index index.html;
+ server_name _;
+ }
+}
+EOF
+
+bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[api]} arvados_workbench_host=http://$localip
+
+exec nginx -c /var/lib/arvados/doc-nginx.conf
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+# Taken from https://github.com/jpetazzo/dind
+
+exec 2>&1
+
+# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver
+dmsetup mknodes
+
+: {LOG:=stdio}
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+[ -d $CGROUP ] || mkdir $CGROUP
+
+if mountpoint -q $CGROUP ; then
+ break
+else
+ mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
+fi
+
+if ! mountpoint -q $CGROUP ; then
+ echo "Could not find or mount cgroups. Tried /sys/fs/cgroup and /cgroup. Did you use --privileged?"
+ exit 1
+fi
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+ mount -t securityfs none /sys/kernel/security || {
+ echo "Could not mount /sys/kernel/security."
+ echo "AppArmor detection and --privileged mode might break."
+ }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+ [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+ mountpoint -q $CGROUP/$SUBSYS ||
+ mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+ # The two following sections address a bug which manifests itself
+ # by a cryptic "lxc-start: no ns_cgroup option specified" when
+ # trying to start containers withina container.
+ # The bug seems to appear when the cgroup hierarchies are not
+ # mounted on the exact same directories in the host, and in the
+ # container.
+
+ # Named, control-less cgroups are mounted with "-o name=foo"
+ # (and appear as such under /proc/<pid>/cgroup) but are usually
+ # mounted on a directory named "foo" (without the "name=" prefix).
+ # Systemd and OpenRC (and possibly others) both create such a
+ # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+ # "name=foo". This shouldn't have any adverse effect.
+ echo $SUBSYS | grep -q ^name= && {
+ NAME=$(echo $SUBSYS | sed s/^name=//)
+ ln -s $SUBSYS $CGROUP/$NAME
+ }
+
+ # Likewise, on at least one system, it has been reported that
+ # systemd would mount the CPU and CPU accounting controllers
+ # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+ # but on a directory called "cpu,cpuacct" (note the inversion
+ # in the order of the groups). This tries to work around it.
+ [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+ echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+ echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+ case "$FD" in
+ # Keep stdin/stdout/stderr
+ [012])
+ ;;
+ # Nuke everything else
+ *)
+ eval exec "$FD>&-"
+ ;;
+ esac
+done
+popd >/dev/null
+
+
+# If a pidfile is still around (for example after a container restart),
+# delete it so that docker can start.
+rm -rf /var/run/docker.pid
+
+read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
+trap "kill -TERM -$pgrp; exit" EXIT TERM KILL SIGKILL SIGTERM SIGQUIT
+
+if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
+ docker daemon $DOCKER_DAEMON_ARGS
+fi
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/git
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+export USER=git
+export USERNAME=git
+export LOGNAME=git
+export HOME=/var/lib/arvados/git
+
+cd ~arvbox
+
+mkdir -p ~arvbox/.ssh ~git/.ssh
+chmod 0700 ~arvbox/.ssh ~git/.ssh
+
+if ! test -s ~arvbox/.ssh/id_rsa ; then
+ ssh-keygen -t rsa -P '' -f .ssh/id_rsa
+ cp ~arvbox/.ssh/id_rsa ~arvbox/.ssh/id_rsa.pub ~git/.ssh
+fi
+
+if test -s ~arvbox/.ssh/known_hosts ; then
+ ssh-keygen -f ".ssh/known_hosts" -R localhost
+fi
+
+if ! test -f /var/lib/arvados/gitolite-setup ; then
+ cd ~git
+
+ # Do a no-op login to populate known_hosts
+ # with the hostkey, so it won't try to ask
+ # about it later.
+ cp .ssh/id_rsa.pub .ssh/authorized_keys
+ ssh -o stricthostkeychecking=no git@localhost true
+ rm .ssh/authorized_keys
+
+ cp /usr/local/lib/arvbox/gitolite.rc .gitolite.rc
+
+ gitolite setup -pk .ssh/id_rsa.pub
+
+ if ! test -d gitolite-admin ; then
+ git clone git@localhost:gitolite-admin
+ fi
+
+ cd gitolite-admin
+ git config user.email arvados
+ git config user.name arvados
+ git config push.default simple
+ git push
+
+ touch /var/lib/arvados/gitolite-setup
+else
+ # Do a no-op login to populate known_hosts
+ # with the hostkey, so it won't try to ask
+ # about it later. Don't run anything,
+ # get the default gitolite behavior.
+ ssh -o stricthostkeychecking=no git@localhost
+fi
+
+prefix=$(arv --format=uuid user current | cut -d- -f1)
+
+if ! test -s /var/lib/arvados/arvados-git-uuid ; then
+ repo_uuid=$(arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}")
+ echo $repo_uuid > /var/lib/arvados/arvados-git-uuid
+fi
+
+repo_uuid=$(cat /var/lib/arvados/arvados-git-uuid)
+
+if ! test -s /var/lib/arvados/arvados-git-link-uuid ; then
+ all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
+
+ set +e
+ read -rd $'\000' newlink <<EOF
+{
+ "tail_uuid":"$all_users_group_uuid",
+ "head_uuid":"$repo_uuid",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF
+ set -e
+ link_uuid=$(arv --format=uuid link create --link "$newlink")
+ echo $link_uuid > /var/lib/arvados/arvados-git-link-uuid
+fi
+
+if ! test -d /var/lib/arvados/git/repositories/$repo_uuid.git ; then
+ git clone --bare /usr/src/arvados /var/lib/arvados/git/repositories/$repo_uuid.git
+else
+ git --git-dir=/var/lib/arvados/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados master:master
+fi
+
+cd /usr/src/arvados/services/api
+export RAILS_ENV=development
+
+git_user_key=$(cat ~git/.ssh/id_rsa.pub)
+
+cat > config/arvados-clients.yml <<EOF
+development:
+ gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
+ gitolite_tmp: /var/lib/arvados/git
+ arvados_api_host: $localip:${services[api]}
+ arvados_api_token: "$ARVADOS_API_TOKEN"
+ arvados_api_host_insecure: true
+ gitolite_arvados_git_user_key: "$git_user_key"
+EOF
+
+while true ; do
+ bundle exec script/arvados-git-sync.rb development
+ sleep 120
+done
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keep-web"
+install bin/keep-web /usr/local/bin
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+exec /usr/local/bin/keep-web -trust-all-content -listen=:${services[keep-web]}
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+sleep 2
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p /var/lib/arvados/gostuff
+cd /var/lib/arvados/gostuff
+
+export GOPATH=$PWD
+mkdir -p "$GOPATH/src/git.curoverse.com"
+ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/keepproxy"
+install bin/keepproxy /usr/local/bin
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+set +e
+read -rd $'\000' keepservice <<EOF
+{
+ "service_host":"$localip",
+ "service_port":${services[keepproxy]},
+ "service_ssl_flag":false,
+ "service_type":"proxy"
+}
+EOF
+set -e
+
+if test -s /var/lib/arvados/keepproxy-uuid ; then
+ keep_uuid=$(cat /var/lib/arvados/keepproxy-uuid)
+ arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
+else
+ UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
+ echo $UUID > /var/lib/arvados/keepproxy-uuid
+fi
+
+exec /usr/local/bin/keepproxy -listen=:${services[keepproxy]}
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+. /usr/local/lib/arvbox/common.sh
+exec /usr/local/lib/arvbox/keep-setup.sh keep0 ${services[keepstore0]}
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+. /usr/local/lib/arvbox/common.sh
+exec /usr/local/lib/arvbox/keep-setup.sh keep1 ${services[keepstore1]}
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh
+
+. /usr/local/lib/arvbox/common.sh
+
+chown -R $PGUSER:$PGGROUP /var/lib/postgresql
+chown -R $PGUSER:$PGGROUP /var/run/postgresql
+chown -R $PGUSER:$PGGROUP /etc/postgresql
+chown -R $PGUSER:$PGGROUP /etc/ssl/private
+
+exec chpst -u $PGUSER:$PGGROUP $0-service
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+if ! test -d /var/lib/postgresql/9.4/main ; then
+ /usr/lib/postgresql/9.4/bin/initdb -D /var/lib/postgresql/9.4/main
+ sh -c "while ! (psql postgres -c'\du' | grep '^ arvbox ') >/dev/null ; do createuser -s arvbox ; sleep 1 ; done" &
+fi
+mkdir -p /var/run/postgresql/9.4-main.pg_stat_tmp
+
+rm -f /var/lib/postgresql/9.4/main/postmaster.pid
+
+exec /usr/lib/postgresql/9.4/bin/postgres -D /var/lib/postgresql/9.4/main -c config_file=/etc/postgresql/9.4/main/postgresql.conf
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+. /usr/local/lib/arvbox/common.sh
+
+set -eu -o pipefail
+
+if ! [[ -d /tmp/arvbox-ready ]] ; then
+ echo
+ echo "Arvados-in-a-box starting"
+ echo
+ echo "Note: if this is a fresh arvbox installation, it may take 10-15 minutes (or longer) to download and"
+ echo "install dependencies. Use \"arvbox log\" to monitor the progress of specific services."
+ echo
+ mkdir -p /tmp/arvbox-ready
+fi
+
+sleep 3
+
+waiting=""
+
+for s in "${!services[@]}"
+do
+ if ! [[ -f /tmp/arvbox-ready/$s ]] ; then
+ if nc -z localhost ${services[$s]} ; then
+ echo "$s is ready at $localip:${services[$s]}"
+ touch /tmp/arvbox-ready/$s
+ else
+ waiting="$waiting $s"
+ fi
+ fi
+done
+
+if ! docker version >/dev/null 2>/dev/null ; then
+ waiting="$waiting docker"
+fi
+
+for sdk_app in arv arv-get cwl-runner arv-mount ; do
+ if ! which $sdk_app >/dev/null ; then
+ waiting="$waiting sdk"
+ break
+ fi
+done
+
+if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
+ waiting="$waiting crunch-dispatch"
+fi
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+
+vm_ok=0
+if test -s /var/lib/arvados/vm-uuid -a -s /var/lib/arvados/superuser_token; then
+ vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+ export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+ if (which arv && arv virtual_machine get --uuid $vm_uuid) >/dev/null 2>/dev/null ; then
+ vm_ok=1
+ fi
+fi
+
+if test $vm_ok = 0 ; then
+ waiting="$waiting vm"
+fi
+
+if ! [[ -z "$waiting" ]] ; then
+ if ps x | grep -v grep | grep "bundle install" > /dev/null; then
+ gemcount=$(ls /var/lib/gems/ruby/2.1.0/gems 2>/dev/null | wc -l)
+
+ gemlockcount=0
+ for l in /usr/src/arvados/services/api/Gemfile.lock \
+ /usr/src/arvados/apps/workbench/Gemfile.lock \
+ /usr/src/sso/Gemfile.lock ; do
+ gc=$(cat $l \
+ | grep -vE "(GEM|PLATFORMS|DEPENDENCIES|$^|remote:|specs:)" \
+ | sed 's/^ *//' | sed 's/(.*)//' | sed 's/ *$//' | sort | uniq | wc -l)
+ gemlockcount=$(($gemlockcount + $gc))
+ done
+ waiting="$waiting (installing ruby gems $gemcount/$gemlockcount)"
+ fi
+
+ if ps x | grep -v grep | grep "c++.*/var/lib/passenger" > /dev/null ; then
+ waiting="$waiting (compiling passenger)"
+ fi
+
+ if ps x | grep -v grep | grep "pip install" > /dev/null; then
+ waiting="$waiting (installing python packages)"
+ fi
+ echo " Waiting for$waiting ..."
+ exit 1
+fi
+
+echo
+echo "Your Arvados-in-a-box is ready!"
+echo "Workbench is running at http://$localip"
+
+rm -r /tmp/arvbox-ready
+
+sv stop ready >/dev/null
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+set -e
+
+/usr/local/lib/arvbox/runsu.sh $0-service
+sv stop sdk
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+mkdir -p ~/.pip /var/lib/arvados/pip
+cat > ~/.pip/pip.conf <<EOF
+[global]
+download_cache = /var/lib/arvados/pip
+EOF
+
+cd /usr/src/arvados/sdk/cli
+run_bundler --binstubs=$PWD/binstubs
+ln -sf /usr/src/arvados/sdk/cli/binstubs/arv /usr/local/bin/arv
+
+cd /usr/src/arvados/sdk/python
+python setup.py sdist
+pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
+
+cd /usr/src/arvados/sdk/cwl
+python setup.py sdist
+pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
+
+cd /usr/src/arvados/services/fuse
+python setup.py sdist
+pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cat > /etc/slurm-llnl/slurm.conf <<EOF
+ControlMachine=$HOSTNAME
+ControlAddr=$HOSTNAME
+AuthType=auth/munge
+DefaultStorageLoc=/var/log/slurm-llnl
+SelectType=select/cons_res
+SelectTypeParameters=CR_CPU_Memory
+SlurmUser=arvbox
+SlurmdUser=arvbox
+SlurmctldPort=7002
+SlurmctldTimeout=300
+SlurmdPort=7003
+SlurmdSpoolDir=/var/tmp/slurmd.spool
+SlurmdTimeout=300
+StateSaveLocation=/var/tmp/slurm.state
+NodeName=$HOSTNAME
+PartitionName=compute State=UP Default=YES Nodes=$HOSTNAME
+EOF
+
+mkdir -p /var/run/munge
+
+/usr/sbin/munged -f
+
+exec /usr/sbin/slurmctld -v -D
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/sbin/slurmd -v -D
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -eux -o pipefail
+
+if ! test -d /var/run/sshd ; then
+ mkdir /var/run/sshd
+ chmod 0755 /var/run/sshd
+fi
+/usr/sbin/sshd -D
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/sso
+export RAILS_ENV=development
+
+run_bundler --without=development
+bundle exec passenger start --runtime-check-only --runtime-dir=/var/lib/passenger
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+set -u
+
+if ! test -s /var/lib/arvados/sso_uuid_prefix ; then
+ ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"' > /var/lib/arvados/sso_uuid_prefix
+fi
+uuid_prefix=$(cat /var/lib/arvados/sso_uuid_prefix)
+
+if ! test -s /var/lib/arvados/sso_secret_token ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_secret_token
+fi
+secret_token=$(cat /var/lib/arvados/sso_secret_token)
+
+if ! test -s /var/lib/arvados/self-signed.key ; then
+ openssl req -new -x509 -nodes -out /var/lib/arvados/self-signed.pem -keyout /var/lib/arvados/self-signed.key -days 365 -subj '/CN=localhost'
+fi
+
+cat >config/application.yml <<EOF
+development:
+ uuid_prefix: $uuid_prefix
+ secret_token: $secret_token
+ default_link_url: "http://$localip"
+ allow_account_registration: true
+EOF
+
+(cd config && /usr/local/lib/arvbox/application_yml_override.py)
+
+if ! test -f /var/lib/arvados/sso_database_pw ; then
+ ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/sso_database_pw
+fi
+database_pw=$(cat /var/lib/arvados/sso_database_pw)
+
+if ! (psql postgres -c "\du" | grep "^ arvados_sso ") >/dev/null ; then
+ psql postgres -c "create user arvados_sso with password '$database_pw'"
+ psql postgres -c "ALTER USER arvados_sso CREATEDB;"
+fi
+
+sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config/database.yml
+
+if ! test -f /var/lib/arvados/sso_database_setup ; then
+ bundle exec rake db:setup
+
+ if ! test -s /var/lib/arvados/sso_app_secret ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_app_secret
+ fi
+ app_secret=$(cat /var/lib/arvados/sso_app_secret)
+
+ bundle exec rails console <<EOF
+c = Client.new
+c.name = "joshid"
+c.app_id = "arvados-server"
+c.app_secret = "$app_secret"
+c.save!
+EOF
+
+ touch /var/lib/arvados/sso_database_setup
+fi
+
+rm -rf tmp
+
+bundle exec rake db:migrate
+
+set +u
+if test "$1" = "--only-setup" ; then
+ exit
+fi
+
+exec bundle exec passenger start --port=${services[sso]} \
+ --runtime-dir=/var/lib/passenger \
+ --ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \
+ --ssl-certificate-key=/var/lib/arvados/self-signed.key
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+set -e
+
+. /usr/local/lib/arvbox/common.sh
+
+git config --system "credential.http://$localip:${services[arv-git-httpd]}/.username" none
+git config --system "credential.http://$localip:${services[arv-git-httpd]}/.helper" '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'
+
+/usr/local/lib/arvbox/runsu.sh $0-service
+
+cd /usr/src/arvados/services/login-sync
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
+
+while true ; do
+ bundle exec arvados-login-sync
+ sleep 120
+done
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+sleep 2
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/arvados/services/login-sync
+run_bundler
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+set -u
+
+export ARVADOS_API_HOST=$localip:${services[api]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
+
+set +e
+read -rd $'\000' vm <<EOF
+{
+ "uuid": "$ARVADOS_VIRTUAL_MACHINE_UUID",
+ "hostname":"$localip"
+}
+EOF
+set -e
+
+if arv virtual_machine get --uuid $ARVADOS_VIRTUAL_MACHINE_UUID ; then
+ arv virtual_machine update --uuid $ARVADOS_VIRTUAL_MACHINE_UUID --virtual-machine "$vm"
+else
+ arv virtual_machine create --virtual-machine "$vm"
+fi
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+set -e
+
+/usr/local/lib/arvbox/runsu.sh $0-service $1
+
+cd /usr/src/arvados/apps/workbench
+
+rm -rf tmp
+mkdir tmp
+chown arvbox:arvbox tmp
+
+if test "$1" != "--only-deps" ; then
+ exec bundle exec passenger start --port 80 \
+ --user arvbox --runtime-dir=/var/lib/passenger
+fi
--- /dev/null
+#!/bin/bash
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+
+cd /usr/src/arvados/apps/workbench
+export RAILS_ENV=development
+
+run_bundler --without=development
+bundle exec passenger start --runtime-check-only --runtime-dir=/var/lib/passenger
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+set -u
+
+if ! test -s /var/lib/arvados/workbench_secret_token ; then
+ ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/workbench_secret_token
+fi
+secret_token=$(cat /var/lib/arvados/workbench_secret_token)
+
+if ! test -s self-signed.key ; then
+ openssl req -new -x509 -nodes -out self-signed.pem -keyout self-signed.key -days 365 -subj '/CN=localhost'
+fi
+
+cat >config/application.yml <<EOF
+development:
+ secret_token: $secret_token
+ arvados_login_base: https://$localip:${services[api]}/login
+ arvados_v1_base: https://$localip:${services[api]}/arvados/v1
+ arvados_insecure_https: true
+ keep_web_download_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
+ keep_web_url: http://$localip:${services[keep-web]}/c=%{uuid_or_pdh}
+ arvados_docsite: http://$localip:${services[doc]}/
+EOF
+
+(cd config && /usr/local/lib/arvbox/application_yml_override.py)
--- /dev/null
+#!/bin/sh
+while ! psql postgres -c\\du >/dev/null 2>/dev/null ; do
+ sleep 1
+done
AVAILABLE_RAM_RATIO = 0.95
+# Workaround datetime.datetime.strptime() thread-safety bug by calling
+# it once before starting threads. https://bugs.python.org/issue7980
+datetime.datetime.strptime('1999-12-31_23:59:59', '%Y-%m-%d_%H:%M:%S')
+
+
class Task(object):
def __init__(self):
self.starttime = None
projects / arvados.git / commitdiff