21230: Activate virtualenv in arvbox doc service

[arvados.git] / tools / arvbox / lib / arvbox / docker / Dockerfile.base

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 115754cac7338cb01737ad8e8ee457682ec7406c..a235274d31e0fc8cd25d8ba1601715a2a54909c1 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -13,19 +13,24 @@ ARG BUILDTYPE
 # tree, and use the $arvados_version commit (passed in via an argument).
 
 ###########################################################################################################
-FROM debian:10-slim as dev
+FROM debian:11-slim as dev
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list
-
 RUN apt-get update && \
     apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    golang -t buster-backports
-
-RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    build-essential ca-certificates git libpam0g-dev
+    build-essential ca-certificates git libpam0g-dev wget
 
 ENV GOPATH /var/lib/gopath
+ARG go_version
+
+# Get Go
+RUN cd /usr/src && \
+    wget https://golang.org/dl/go${go_version}.linux-amd64.tar.gz && \
+    tar xzf go${go_version}.linux-amd64.tar.gz && \
+    ln -s /usr/src/go/bin/go /usr/local/bin/go-${go_version} && \
+    ln -s /usr/src/go/bin/gofmt /usr/local/bin/gofmt-${go_version} && \
+    ln -s /usr/local/bin/go-${go_version} /usr/local/bin/go && \
+    ln -s /usr/local/bin/gofmt-${go_version} /usr/local/bin/gofmt
 
 # the --mount option requires the experimental syntax enabled (enables
 # buildkit) on the first line of this file. This Dockerfile must also be built
@@ -37,19 +42,23 @@ RUN --mount=type=bind,target=/usr/src/arvados \
     go install
 
 ###########################################################################################################
-FROM debian:10-slim as demo
+FROM debian:11-slim as demo
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list
-
 RUN apt-get update && \
     apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    golang -t buster-backports
-
-RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    build-essential ca-certificates git libpam0g-dev
+    build-essential ca-certificates git libpam0g-dev wget
 
 ENV GOPATH /var/lib/gopath
+ARG go_version
+
+RUN cd /usr/src && \
+    wget https://golang.org/dl/go${go_version}.linux-amd64.tar.gz && \
+    tar xzf go${go_version}.linux-amd64.tar.gz && \
+    ln -s /usr/src/go/bin/go /usr/local/bin/go-${go_version} && \
+    ln -s /usr/src/go/bin/gofmt /usr/local/bin/gofmt-${go_version} && \
+    ln -s /usr/local/bin/go-${go_version} /usr/local/bin/go && \
+    ln -s /usr/local/bin/gofmt-${go_version} /usr/local/bin/gofmt
 
 ARG arvados_version
 RUN echo arvados_version is git commit $arvados_version
@@ -66,14 +75,14 @@ RUN cd /usr/src && \
 FROM ${BUILDTYPE} as base
 
 ###########################################################################################################
-FROM debian:10
+FROM debian:11
 ENV DEBIAN_FRONTEND noninteractive
 
 # The arvbox-specific dependencies are
-#  gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less
+#  gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less
 RUN apt-get update && \
     apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \
+    gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less vim-tiny && \
     apt-get clean
 
 ENV GOPATH /var/lib/gopath
@@ -84,14 +93,22 @@ COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server
 RUN $GOPATH/bin/arvados-server --version
 RUN $GOPATH/bin/arvados-server install -type test
 
+# Set up a virtualenv for all Python tools in arvbox.
+# This is used mainly by the `sdk` service, but `doc` and internal scripts
+# also rely on it.
+# 1. Install wheel just to modernize the virtualenv.
+# 2. Install setuptools as an sdk build dependency; PyYAML for all tests
+#    and yml_override;py; and pdoc for the doc service.
+# Everything else is installed by the sdk service on boot.
+RUN python3 -m venv /opt/arvados-py \
+ && /opt/arvados-py/bin/pip install --no-cache-dir wheel \
+ && /opt/arvados-py/bin/pip install --no-cache-dir setuptools PyYAML pdoc
+
 RUN /etc/init.d/postgresql start && \
     su postgres -c 'dropuser arvados' && \
     su postgres -c 'createuser -s arvbox' && \
     /etc/init.d/postgresql stop
 
-ENV GEM_HOME /var/lib/arvados/lib/ruby/gems/2.5.0
-ENV PATH $PATH:$GEM_HOME/bin
-
 VOLUME /var/lib/docker
 VOLUME /var/log/nginx
 VOLUME /etc/ssl/private
@@ -102,11 +119,12 @@ ADD $workdir/8D81803C0EBFCD88.asc /tmp/
 RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \
     rm -f /tmp/8D81803C0EBFCD88.asc
 
-RUN mkdir -p /etc/apt/sources.list.d && \
-    echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
-    apt-get update && \
-    apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \
-    apt-get clean
+# docker is now installed by arvados-server install
+# RUN mkdir -p /etc/apt/sources.list.d && \
+#     echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
+#     apt-get update && \
+#     apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \
+#     apt-get clean
 
 # Set UTF-8 locale
 RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen
@@ -133,6 +151,8 @@ COPY $workdir/runit /etc/runit
 # put everything (/var/lib/arvados)
 ENV ARVADOS_CONTAINER_PATH /var/lib/arvados-arvbox
 
+RUN /bin/ln -s /var/lib/arvados/bin/ruby /usr/local/bin/
+
 # Start the supervisor.
 ENV SVDIR /etc/service
 STOPSIGNAL SIGINT