Merge branch '10025-arvbox-layers' closes #10025

diff --git a/build/run-build-docker-images.sh b/build/run-build-docker-images.sh
index d03169457e7d3a1a33a535e50deed79b3f398828..a7dc30cfaedce168fab2ac3ec2d70d3e70bf297b 100755 (executable)
--- a/build/run-build-docker-images.sh
+++ b/build/run-build-docker-images.sh
@@ -125,7 +125,7 @@ timer_reset
 # clean up the docker build environment
 cd "$WORKSPACE"
 
-tools/arvbox/bin/arvbox rebuild localdemo
+tools/arvbox/bin/arvbox build localdemo
 ECODE=$?
 
 if [[ "$ECODE" != "0" ]]; then

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 2ebe13c895eb16a775ca247b028a60cdc6eba59c..54f5b82e652398ea09811ee85b2ec7e54f19067b 100755 (executable)
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -104,6 +104,8 @@ run() {
     CONFIG=$1
     TAG=$2
 
+    shift
+
     if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
         echo "Container $ARVBOX_CONTAINER is already running"
         exit 0
@@ -113,10 +115,15 @@ run() {
         echo "Container $ARVBOX_CONTAINER already exists but is not running; use restart or rebuild"
         exit 1
     fi
-   
+
     if test ! -z "$TAG"
     then
-       TAG=":$TAG"
+        if test $(echo $TAG | cut -c1-1) != '-' ; then
+           TAG=":$TAG"
+            shift
+        else
+            unset TAG
+        fi
     fi
 
     if echo "$CONFIG" | grep '^public' ; then
@@ -260,11 +267,14 @@ build() {
         echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
         exit 1
     fi
-    docker build $NO_CACHE -t arvados/arvbox-base -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
+    GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD)
+    docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
     if test "$1" = localdemo -o "$1" = publicdemo ; then
-        docker build $NO_CACHE -t arvados/arvbox-demo -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+        docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+        docker tag -f arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest
     else
-        docker build $NO_CACHE -t arvados/arvbox-dev -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+        docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+        docker tag -f arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest
     fi
 }
 
@@ -397,7 +407,7 @@ case "$subcmd" in
 
     cat)
         if test -n "$1" ; then
-            exec docker exec -ti $ARVBOX_CONTAINER cat "$@"
+            exec docker exec $ARVBOX_CONTAINER cat "$@"
         else
             echo "Usage: $0 $subcmd <files>"
         fi

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 9dee8847cf57bee6a8115df73f3e21d210bec79a..0b926732b656a3b435cef6e1061887a622e77997 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -16,10 +16,15 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
     libjson-perl nginx gitolite3 lsof \
     apt-transport-https ca-certificates slurm-wlm
 
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
+    linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
+
 RUN cd /usr/local && \
-    curl -O http://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz && \
-    tar -xzf go1.6.2.linux-amd64.tar.gz && \
-    rm go1.6.2.linux-amd64.tar.gz && \
+    GOVERSION=1.7.1 && \
+    curl -O http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz && \
+    tar -xzf go${GOVERSION}.linux-amd64.tar.gz && \
+    rm go${GOVERSION}.linux-amd64.tar.gz && \
     cd bin && \
     ln -s /usr/local/go/bin/* .
 
@@ -41,6 +46,15 @@ RUN cd /root && \
     GOPATH=$PWD go get github.com/curoverse/runsvinit && \
     install bin/runsvinit /usr/local/bin
 
+RUN set -e && \
+ PJS=phantomjs-1.9.7-linux-x86_64 && \
+ curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
+ tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
+ ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+
+ARG arvados_version
+RUN echo arvados_version is git commit $arvados_version
+
 ADD fuse.conf /etc/
 
 ADD crunch-setup.sh gitolite.rc \

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 57105ea88bf4a58bf397e0b2fd74c1d520afee06..d58719b61d4c15a33d3ad41aaa5b58ec9f843bc9 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -1,5 +1,5 @@
 FROM arvados/arvbox-base
-ARG arvados_version=master
+ARG arvados_version
 ARG sso_version=master
 
 RUN cd /usr/src && \

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
index e68642fecd5a2f0ff343eb869d569a43b945cd6a..f99551e786363bf135d8413c09982187bf85c4f4 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
@@ -1,14 +1,5 @@
 FROM arvados/arvbox-base
-
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
-    linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
-
-RUN set -e && \
- PJS=phantomjs-1.9.7-linux-x86_64 && \
- curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
- tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
- ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+ARG arvados_version
 
 ADD service/ /var/lib/arvbox/service
 RUN rmdir /etc/service && ln -sf /var/lib/arvbox/service /etc

diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 3733fa2ecb187b7a1daa5ce7851546faf5856603..742658f5f1d797bdbaa068e3b029445b545b946f 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -45,12 +45,7 @@ run_bundler() {
 
 pip_install() {
     pushd /var/lib/pip
-    for p in $(ls http*.tar.gz) ; do
-        if test -f $p ; then
-            ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
-        fi
-    done
-    for p in $(ls http*.whl) ; do
+    for p in $(ls http*.tar.gz) $(ls http*.whl) $(ls http*.zip) ; do
         if test -f $p ; then
             ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
         fi

diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 29452ab9943c7853da919fa130de0b5690e249ba..8afd2c8561e1250b741e9d32a69847f92f77abab 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -19,10 +19,10 @@ cd /usr/src/arvados/sdk/python
 python setup.py sdist
 pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
 
-cd /usr/src/arvados/sdk/cwl
-python setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
-
 cd /usr/src/arvados/services/fuse
 python setup.py sdist
 pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
+
+cd /usr/src/arvados/sdk/cwl
+python setup.py sdist
+pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)