From b27c53dedf632f614356305bc624befa5477b98e Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Thu, 24 Sep 2020 20:49:57 -0400 Subject: [PATCH] 16267: implement review feedback. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- tools/arvbox/bin/arvbox | 17 +++- .../arvbox/lib/arvbox/docker/Dockerfile.base | 93 +++++++++++++++---- .../arvbox/lib/arvbox/docker/Dockerfile.demo | 1 + tools/arvbox/lib/arvbox/docker/common.sh | 2 + 4 files changed, 93 insertions(+), 20 deletions(-) diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox index 8fd77f1f2d..e8e3fa17ce 100755 --- a/tools/arvbox/bin/arvbox +++ b/tools/arvbox/bin/arvbox @@ -363,6 +363,7 @@ stop() { } build() { + export DOCKER_BUILDKIT=1 if ! test -f "$ARVBOX_DOCKER/Dockerfile.base" ; then echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)" exit 1 @@ -373,12 +374,24 @@ build() { FORCE=-f fi GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD) - docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER" - docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest if test "$1" = localdemo -o "$1" = publicdemo ; then + set +e + if which greadlink >/dev/null 2>/dev/null ; then + ARVADOS_ROOT=$(greadlink -f $(dirname $0)/../../../) + else + ARVADOS_ROOT=$(readlink -f $(dirname $0)/../../../) + fi + set -e + docker build --build-arg=BUILDTYPE=demo $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVADOS_ROOT" + docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER" docker tag $FORCE arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest else + if ! test -d "$ARVADOS_ROOT" ; then + git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT" + fi + docker build --build-arg=BUILDTYPE=dev $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVADOS_ROOT" + docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER" docker tag $FORCE arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest fi diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base index 9031de79bb..115754cac7 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base @@ -1,9 +1,19 @@ +# syntax = docker/dockerfile:experimental # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 -FROM debian:10 +ARG BUILDTYPE + +# We're using poor man's conditionals (see +# https://www.docker.com/blog/advanced-dockerfiles-faster-builds-and-smaller-images-using-buildkit-and-multistage-builds/) +# here to dtrt in the dev/test scenario and the demo scenario. In the dev/test +# scenario, we use the docker context (i.e. the copy of Arvados checked out on +# the host) to build arvados-server. In the demo scenario, we check out a new +# tree, and use the $arvados_version commit (passed in via an argument). +########################################################################################################### +FROM debian:10-slim as dev ENV DEBIAN_FRONTEND noninteractive RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list @@ -12,22 +22,67 @@ RUN apt-get update && \ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ golang -t buster-backports -# The arvbox-specific dependencies are -# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ - build-essential ca-certificates git libpam0g-dev \ - gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \ - apt-get clean + build-essential ca-certificates git libpam0g-dev ENV GOPATH /var/lib/gopath +# the --mount option requires the experimental syntax enabled (enables +# buildkit) on the first line of this file. This Dockerfile must also be built +# with the DOCKER_BUILDKIT=1 environment variable set. +RUN --mount=type=bind,target=/usr/src/arvados \ + cd /usr/src/arvados && \ + go mod download && \ + cd cmd/arvados-server && \ + go install + +########################################################################################################### +FROM debian:10-slim as demo +ENV DEBIAN_FRONTEND noninteractive + +RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list + +RUN apt-get update && \ + apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + golang -t buster-backports + +RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + build-essential ca-certificates git libpam0g-dev + +ENV GOPATH /var/lib/gopath + +ARG arvados_version +RUN echo arvados_version is git commit $arvados_version + RUN cd /usr/src && \ - git clone https://git.arvados.org/arvados.git && \ - cd arvados && \ + git clone --no-checkout https://git.arvados.org/arvados.git && \ + git -C arvados checkout ${arvados_version} && \ + cd /usr/src/arvados && \ go mod download && \ cd cmd/arvados-server && \ - go install && \ - $GOPATH/bin/arvados-server install -type test + go install + +########################################################################################################### +FROM ${BUILDTYPE} as base + +########################################################################################################### +FROM debian:10 +ENV DEBIAN_FRONTEND noninteractive + +# The arvbox-specific dependencies are +# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less +RUN apt-get update && \ + apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \ + apt-get clean + +ENV GOPATH /var/lib/gopath +RUN echo buildtype is $BUILDTYPE + +RUN mkdir -p $GOPATH/bin/ +COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server +RUN $GOPATH/bin/arvados-server --version +RUN $GOPATH/bin/arvados-server install -type test RUN /etc/init.d/postgresql start && \ su postgres -c 'dropuser arvados' && \ @@ -41,7 +96,9 @@ VOLUME /var/lib/docker VOLUME /var/log/nginx VOLUME /etc/ssl/private -ADD 8D81803C0EBFCD88.asc /tmp/ +ARG workdir + +ADD $workdir/8D81803C0EBFCD88.asc /tmp/ RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \ rm -f /tmp/8D81803C0EBFCD88.asc @@ -60,16 +117,16 @@ ENV LC_ALL en_US.UTF-8 ARG arvados_version RUN echo arvados_version is git commit $arvados_version -ADD fuse.conf /etc/ +COPY $workdir/fuse.conf /etc/ -ADD gitolite.rc \ - keep-setup.sh common.sh createusers.sh \ - logger runsu.sh waitforpostgres.sh \ - yml_override.py api-setup.sh \ - go-setup.sh devenv.sh cluster-config.sh edit_users.py \ +COPY $workdir/gitolite.rc \ + $workdir/keep-setup.sh $workdir/common.sh $workdir/createusers.sh \ + $workdir/logger $workdir/runsu.sh $workdir/waitforpostgres.sh \ + $workdir/yml_override.py $workdir/api-setup.sh \ + $workdir/go-setup.sh $workdir/devenv.sh $workdir/cluster-config.sh $workdir/edit_users.py \ /usr/local/lib/arvbox/ -ADD runit /etc/runit +COPY $workdir/runit /etc/runit # arvbox mounts a docker volume at $ARVADOS_CONTAINER_PATH, make sure that that # doesn't overlap with the directory where `arvados-server install -type test` diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo index 192b2a144c..777c71356c 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo @@ -8,6 +8,7 @@ ARG composer_version=arvados-fork ARG workbench2_version=master RUN cd /usr/src && \ + git clone --no-checkout https://git.arvados.org/arvados.git && \ git -C arvados checkout ${arvados_version} && \ git -C arvados pull && \ git clone --no-checkout https://github.com/arvados/composer.git && \ diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh index 4bfe9dd513..48d3566405 100644 --- a/tools/arvbox/lib/arvbox/docker/common.sh +++ b/tools/arvbox/lib/arvbox/docker/common.sh @@ -77,6 +77,8 @@ run_bundler() { # flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion} # fi # fi + # Make sure to put the gem binaries in the right place + flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@" fi -- 2.30.2