10025: Rearrange arvbox layers for better caching. Embed build version to

ensure that upper layers rebuild but lower layers do not.

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 2ebe13c895eb16a775ca247b028a60cdc6eba59c..60917b5bd110004b151e27a0d82e6f45bcc14557 100755 (executable)
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -113,9 +113,10 @@ run() {
         echo "Container $ARVBOX_CONTAINER already exists but is not running; use restart or rebuild"
         exit 1
     fi
-   
+
     if test ! -z "$TAG"
     then
+        # XXX should push tag back onto argument list if it starts with '-'
        TAG=":$TAG"
     fi
 
@@ -209,6 +210,7 @@ run() {
                    /usr/local/lib/arvbox/runsu.sh \
                    /var/lib/arvbox/service/api/run-service --only-setup
 
+            shift ; shift
             docker exec -ti \
                    $ARVBOX_CONTAINER \
                    /usr/local/lib/arvbox/runsu.sh \
@@ -260,11 +262,14 @@ build() {
         echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
         exit 1
     fi
-    docker build $NO_CACHE -t arvados/arvbox-base -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
+    GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD)
+    docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
     if test "$1" = localdemo -o "$1" = publicdemo ; then
-        docker build $NO_CACHE -t arvados/arvbox-demo -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+        docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+        docker tag arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest
     else
-        docker build $NO_CACHE -t arvados/arvbox-dev -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+        docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+        docker tag arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest
     fi
 }
 
@@ -397,7 +402,7 @@ case "$subcmd" in
 
     cat)
         if test -n "$1" ; then
-            exec docker exec -ti $ARVBOX_CONTAINER cat "$@"
+            exec docker exec $ARVBOX_CONTAINER cat "$@"
         else
             echo "Usage: $0 $subcmd <files>"
         fi

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 9dee8847cf57bee6a8115df73f3e21d210bec79a..30756c923cd6626aaf3f76b7e22698492d89cbad 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -16,10 +16,15 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
     libjson-perl nginx gitolite3 lsof \
     apt-transport-https ca-certificates slurm-wlm
 
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
+    linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
+
 RUN cd /usr/local && \
-    curl -O http://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz && \
-    tar -xzf go1.6.2.linux-amd64.tar.gz && \
-    rm go1.6.2.linux-amd64.tar.gz && \
+    GOVERSION=1.7.1 && \
+    curl -O http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz && \
+    tar -xzf go${GOVERSION}.linux-amd64.tar.gz && \
+    rm go${GOVERSION}.linux-amd64.tar.gz && \
     cd bin && \
     ln -s /usr/local/go/bin/* .
 
@@ -41,6 +46,15 @@ RUN cd /root && \
     GOPATH=$PWD go get github.com/curoverse/runsvinit && \
     install bin/runsvinit /usr/local/bin
 
+RUN set -e && \
+ PJS=phantomjs-1.9.7-linux-x86_64 && \
+ curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
+ tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
+ ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+
+ARG arvados_version
+RUN echo arvados_version
+
 ADD fuse.conf /etc/
 
 ADD crunch-setup.sh gitolite.rc \

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 57105ea88bf4a58bf397e0b2fd74c1d520afee06..a8431f9b2d83601e96529024d4bed122d813f333 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -1,5 +1,4 @@
 FROM arvados/arvbox-base
-ARG arvados_version=master
 ARG sso_version=master
 
 RUN cd /usr/src && \

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
index e68642fecd5a2f0ff343eb869d569a43b945cd6a..99db79798094199d61851c8ab2ffcf4941e6bf0f 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev
@@ -1,15 +1,5 @@
 FROM arvados/arvbox-base
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
-    linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
-
-RUN set -e && \
- PJS=phantomjs-1.9.7-linux-x86_64 && \
- curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
- tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
- ln -s ../$PJS/bin/phantomjs /usr/local/bin/
-
 ADD service/ /var/lib/arvbox/service
 RUN rmdir /etc/service && ln -sf /var/lib/arvbox/service /etc
 

diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 3733fa2ecb187b7a1daa5ce7851546faf5856603..742658f5f1d797bdbaa068e3b029445b545b946f 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -45,12 +45,7 @@ run_bundler() {
 
 pip_install() {
     pushd /var/lib/pip
-    for p in $(ls http*.tar.gz) ; do
-        if test -f $p ; then
-            ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
-        fi
-    done
-    for p in $(ls http*.whl) ; do
+    for p in $(ls http*.tar.gz) $(ls http*.whl) $(ls http*.zip) ; do
         if test -f $p ; then
             ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
         fi

diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 29452ab9943c7853da919fa130de0b5690e249ba..8afd2c8561e1250b741e9d32a69847f92f77abab 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -19,10 +19,10 @@ cd /usr/src/arvados/sdk/python
 python setup.py sdist
 pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
 
-cd /usr/src/arvados/sdk/cwl
-python setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
-
 cd /usr/src/arvados/services/fuse
 python setup.py sdist
 pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
+
+cd /usr/src/arvados/sdk/cwl
+python setup.py sdist
+pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)