end
within('.recent-processes') do
- assert_text 'pipeline_with_job'
+ assert_text 'running'
within('.row-zzzzz-xvhdp-cr4runningcntnr') do
assert_text 'requester_for_running_cr'
run-build-docker-images.sh Build arvbox Docker images.
-run-build-docker-jobs-image.sh Build arvados/jobs Docker image.
+run-build-docker-jobs-image.sh Build arvados/jobs Docker image
+ (uses published debian packages)
+
+build-dev-docker-jobs-image.sh Build developer arvados/jobs Docker image
+ (uses local git tree)
run-library.sh A library of functions shared by the
various scripts in this
--- /dev/null
+#!/bin/bash
+
+read -rd "\000" helpmessage <<EOF
+Build an arvados/jobs Docker image from local git tree.
+
+Intended for use by developers working on arvados-python-client or
+arvados-cwl-runner and need to run a crunch job with a custom package
+version. Also supports building custom cwltool if CWLTOOL is set.
+
+Syntax:
+ WORKSPACE=/path/to/arvados $(basename $0)
+
+WORKSPACE=path Path to the Arvados source tree to build packages from
+CWLTOOL=path (optional) Path to cwltool git repository.
+
+EOF
+
+set -e
+
+if [[ -z "$WORKSPACE" ]] ; then
+ echo "$helpmessage"
+ echo
+ echo "Must set WORKSPACE"
+ exit 1
+fi
+
+if [[ -z "$ARVADOS_API_HOST" || -z "$ARVADOS_API_TOKEN" ]] ; then
+ echo "$helpmessage"
+ echo
+ echo "Must set ARVADOS_API_HOST and ARVADOS_API_TOKEN"
+ exit 1
+fi
+
+cd "$WORKSPACE"
+
+(cd sdk/python && python setup.py sdist)
+sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
+
+(cd sdk/cwl && python setup.py sdist)
+runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
+
+rm -rf sdk/cwl/cwltool_dist
+mkdir -p sdk/cwl/cwltool_dist
+if [[ -n "$CWLTOOL" ]] ; then
+ (cd "$CWLTOOL" && python setup.py sdist)
+ cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
+ cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
+fi
+
+. build/run-library.sh
+
+python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/python)
+else
+ gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/cwl)
+fi
+
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
+echo arv-keepdocker arvados/jobs $gittag
+arv-keepdocker arvados/jobs $gittag
fpm_build cwltest "" "" python 1.0.20160907111242
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-fpm_build cwltool "" "" python 1.0.20161128202906
+cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(1\.0\..*\)'.*/\1/")
+fpm_build cwltool "" "" python $cwltoolversion
# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
fpm_build rdflib-jsonld "" "" python 0.3.0
}, ensure_unique_name=True).execute(
num_retries=self.arvrunner.num_retries)
- done_outputs(self, record, tmpdir, outdir, keepdir)
+ return done_outputs(self, record, tmpdir, outdir, keepdir)
def done_outputs(self, record, tmpdir, outdir, keepdir):
self.builder.outdir = outdir
--- /dev/null
+# Dockerfile for building an arvados/jobs Docker image from local git tree.
+#
+# Intended for use by developers working on arvados-python-client or
+# arvados-cwl-runner and need to run a crunch job with a custom package
+# version.
+#
+# Use arvados/build/build-dev-docker-jobs-image.sh to build.
+#
+# (This dockerfile file must be located in the arvados/sdk/ directory because
+# of the docker build root.)
+
+FROM debian:jessie
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs
+
+RUN pip install -U setuptools
+
+ARG sdk
+ARG runner
+ARG cwltool
+
+ADD python/dist/$sdk /tmp/
+ADD cwl/cwltool_dist/$cwltool /tmp/
+ADD cwl/dist/$runner /tmp/
+
+RUN cd /tmp/arvados-python-client-* && python setup.py install
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi
+RUN cd /tmp/arvados-cwl-runner-* && python setup.py install
+
+# Install dependencies and set up system.
+RUN /usr/sbin/adduser --disabled-password \
+ --gecos 'Crunch execution user' crunch && \
+ /usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job
+
+USER crunch
--- /dev/null
+#!/bin/bash
+# bash functions for managing Arvados tokens and other conveniences.
+
+read -rd "\000" helpmessage <<EOF
+$(basename $0): bash functions for managing Arvados tokens and other shortcuts.
+
+Syntax:
+ . $0 # activate for current shell
+ $0 --install # install into .bashrc
+
+arvswitch <name>
+ Set ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment based on
+ $HOME/.config/arvados/<name>.conf
+ With no arguments, list available Arvados configurations.
+
+arvsave <name>
+ Save values of ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment to
+ $HOME/.config/arvados/<name>.conf
+
+arvrm <name>
+ Delete $HOME/.config/arvados/<name>.conf
+
+arvboxswitch <name>
+ Set ARVBOX_CONTAINER to <name>
+ With no arguments, list available arvboxes.
+
+arvopen:
+ Open an Arvados uuid in web browser (http://curover.se)
+
+arvissue
+ Open an Arvados ticket in web browser (http://dev.arvados.org)
+
+EOF
+
+if [[ "$1" = "--install" ]] ; then
+ this=$(readlink -f $0)
+ if ! grep ". $this" ~/.bashrc >/dev/null ; then
+ echo ". $this" >> ~/.bashrc
+ echo "Installed into ~/.bashrc"
+ else
+ echo "Already installed in ~/.bashrc"
+ fi
+elif ! [[ $0 =~ bash$ ]] ; then
+ echo "$helpmessage"
+fi
+
+HISTIGNORE=$HISTIGNORE:'export ARVADOS_API_TOKEN=*'
+
+arvswitch() {
+ if [[ -n "$1" ]] ; then
+ if [[ -f $HOME/.config/arvados/$1.conf ]] ; then
+ unset ARVADOS_API_HOST_INSECURE
+ for a in $(cat $HOME/.config/arvados/$1.conf) ; do export $a ; done
+ echo "Switched to $1"
+ else
+ echo "$1 unknown"
+ fi
+ else
+ echo "Switch Arvados environment conf"
+ echo "Usage: arvswitch name"
+ echo "Available confs:" $((cd $HOME/.config/arvados && ls --indicator-style=none *.conf) | rev | cut -c6- | rev)
+ fi
+}
+
+arvsave() {
+ if [[ -n "$1" ]] ; then
+ touch $HOME/.config/arvados/$1.conf
+ chmod 0600 $HOME/.config/arvados/$1.conf
+ env | grep ARVADOS_ > $HOME/.config/arvados/$1.conf
+ else
+ echo "Save current Arvados environment variables to conf file"
+ echo "Usage: arvsave name"
+ fi
+}
+
+arvrm() {
+ if [[ -n "$1" ]] ; then
+ if [[ -f $HOME/.config/arvados/$1.conf ]] ; then
+ rm $HOME/.config/arvados/$1.conf
+ else
+ echo "$1 unknown"
+ fi
+ else
+ echo "Delete Arvados environment conf"
+ echo "Usage: arvrm name"
+ fi
+}
+
+arvboxswitch() {
+ if [[ -n "$1" ]] ; then
+ if [[ -d $HOME/.arvbox/$1 ]] ; then
+ export ARVBOX_CONTAINER=$1
+ echo "Arvbox switched to $1"
+ else
+ echo "$1 unknown"
+ fi
+ else
+ if test -z "$ARVBOX_CONTAINER" ; then
+ ARVBOX_CONTAINER=arvbox
+ fi
+ echo "Switch Arvbox environment conf"
+ echo "Usage: arvboxswitch name"
+ echo "Your current container is: $ARVBOX_CONTAINER"
+ echo "Available confs:" $(cd $HOME/.arvbox && ls --indicator-style=none)
+ fi
+}
+
+arvopen() {
+ if [[ -n "$1" ]] ; then
+ xdg-open https://curover.se/$1
+ else
+ echo "Open Arvados uuid in browser"
+ echo "Usage: arvopen uuid"
+ fi
+}
+
+arvissue() {
+ if [[ -n "$1" ]] ; then
+ xdg-open https://dev.arvados.org/issues/$1
+ else
+ echo "Open Arvados issue in browser"
+ echo "Usage: arvissue uuid"
+ fi
+}
export RAILS_ENV=development
run_bundler --without=development
-bundle exec passenger start --runtime-check-only --runtime-dir=/var/lib/passenger
+bundle exec passenger-config build-native-support
+bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
fi
exec bundle exec passenger start --port=${services[api]} \
- --runtime-dir=/var/lib/passenger \
--ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \
--ssl-certificate-key=/var/lib/arvados/self-signed.key
projects / arvados.git / commitdiff