21230: Use virtualenv in arvbox

This is preparation to let us build arvbox on Debian 12, which has
adopted PEP 668 and doesn't permit installing packages outside a
virtualenv anymore.

This does move pdoc installation out of the doc service and into the
Dockerfile. The doc code currently doesn't have any way to version this
dependency, and it rarely changes in development, so this seems fine as
a code simplification.

All the code that tries to "pip install from cache only, then use the
network" is gone because as best I can tell it's already non-functional:
the pip cache isn't structured in the way that it assumes
anymore. Generally pip looks at what's installed in the virtualenv and
only installs what it *must* to satisfy dependencies, so this seems
fine.

Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 778b752fd40a8d740a137c2c19c85c855aef4a0a..6a03b303f7a4fa54262bb86081fe7cd4f8ad9f22 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -79,10 +79,10 @@ FROM debian:11
 ENV DEBIAN_FRONTEND noninteractive
 
 # The arvbox-specific dependencies are
-#  gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less
+#  gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less
 RUN apt-get update && \
     apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less vim-tiny && \
+    gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less vim-tiny && \
     apt-get clean
 
 ENV GOPATH /var/lib/gopath
@@ -93,6 +93,18 @@ COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server
 RUN $GOPATH/bin/arvados-server --version
 RUN $GOPATH/bin/arvados-server install -type test
 
+# Set up a virtualenv for all Python tools in arvbox.
+# This is used mainly by the `sdk` service, but `doc` and internal scripts
+# also rely on it.
+# 1. Install wheel just to modernize the virtualenv.
+# 2. Install setuptools as an sdk build dependency; PyYAML for all tests
+#    and yml_override;py; and pdoc for the doc service.
+# Everything else is installed by the sdk service on boot.
+RUN python3 -m venv /opt/arvados-py \
+ && /opt/arvados-py/bin/pip install --no-cache-dir wheel \
+ && /opt/arvados-py/bin/pip install --no-cache-dir setuptools PyYAML pdoc \
+ && ln -s /opt/arvados-py/bin/pdoc /usr/local/bin/
+
 RUN /etc/init.d/postgresql start && \
     su postgres -c 'dropuser arvados' && \
     su postgres -c 'createuser -s arvbox' && \

diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 9c5df83c0e91b25f523531a5512b7efa00c2370f..81516ef08a5d50bf5bb58a501329edf20d0dd646 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -6,7 +6,7 @@ export RUBY_VERSION=3.2.2
 export BUNDLER_VERSION=2.4.22
 
 export DEBIAN_FRONTEND=noninteractive
-export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
 export npm_config_cache=/var/lib/npm
 export npm_config_cache_min=Infinity
 export R_LIBS=/var/lib/Rlibs
@@ -100,24 +100,3 @@ bundler_binstubs() {
     fi
     flock $GEMLOCK $BUNDLER binstubs --all
 }
-
-PYCMD=""
-pip_install() {
-    pushd /var/lib/pip
-    for p in $(ls http*.tar.gz) $(ls http*.tar.bz2) $(ls http*.whl) $(ls http*.zip) ; do
-        if test -f $p ; then
-            ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
-        fi
-    done
-    popd
-
-    if [ "$PYCMD" = "python3" ]; then
-        if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
-            pip3 install --prefix /usr/local $1
-        fi
-    else
-        if ! pip install --no-index --find-links /var/lib/pip $1 ; then
-            pip install $1
-        fi
-    fi
-}

diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh
index 4cafd8c09c2f6fbbd0758b53a9a0a1368765159d..9224b80f52dafa5321d4f85fcd38840f6cfa8c5b 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/createusers.sh
+++ b/tools/arvbox/lib/arvbox/docker/createusers.sh
@@ -14,6 +14,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
     mkdir -p $ARVADOS_CONTAINER_PATH/git \
           /var/lib/passenger /var/lib/gopath \
           /var/lib/pip /var/lib/npm
+    /opt/arvados-py/bin/pip config --site set global.cache-dir /var/lib/pip
 
     if test -z "$ARVBOX_HOME" ; then
         ARVBOX_HOME=$ARVADOS_CONTAINER_PATH
@@ -31,7 +32,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
     useradd --groups docker crunch
 
     if [[ "$1" != --no-chown ]] ; then
-        chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH \
+        chown arvbox:arvbox -R /usr/local /opt/arvados-py $ARVADOS_CONTAINER_PATH \
               /var/lib/passenger /var/lib/postgresql \
               /var/lib/nginx /var/log/nginx /etc/ssl/private \
               /var/lib/gopath /var/lib/pip /var/lib/npm
@@ -43,7 +44,7 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
     echo "arvbox    ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
 
     cat <<EOF > /etc/profile.d/paths.sh
-export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
 export npm_config_cache=/var/lib/npm
 export npm_config_cache_min=Infinity
 export R_LIBS=/var/lib/Rlibs

diff --git a/tools/arvbox/lib/arvbox/docker/edit_users.py b/tools/arvbox/lib/arvbox/docker/edit_users.py
index ab046b11d42751d3939cc4aaee6cba73f055f598..cb44b984b72d72641551f7c5a0a71efdd3cdfed5 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/edit_users.py
+++ b/tools/arvbox/lib/arvbox/docker/edit_users.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/opt/arvados-py/bin/python3
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: AGPL-3.0

diff --git a/tools/arvbox/lib/arvbox/docker/service/doc/run-service b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
index c40e1175edbe9e55326109cb666c630c4eda621e..8d3fdc836573f1fe99813c5e82a2080607517294 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/doc/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
@@ -18,13 +18,10 @@ cd /usr/src/arvados/doc
 run_bundler --without=development
 
 # Generating the Python and R docs is expensive, so for development if the file
-# "no-sdk" exists then skip the Python and R stuff.
+# "no-sdk" exists then skip installing R stuff.
 if [[ ! -f /usr/src/arvados/doc/no-sdk ]] ; then
     cd /usr/src/arvados/sdk/R
     R --quiet --vanilla --file=install_deps.R
-
-    export PYCMD=python3
-    pip_install pdoc
 fi
 
 if test "$1" = "--only-deps" ; then

diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
index 5bff5610529d43688340d0181ae5f8342437709f..218c6361ba75633f494482c77fb6be6ebe343c80 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service
@@ -8,12 +8,6 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p ~/.pip /var/lib/pip
-cat > ~/.pip/pip.conf <<EOF
-[global]
-download_cache = /var/lib/pip
-EOF
-
 cd /usr/src/arvados/sdk/ruby
 run_bundler
 bundler_binstubs
@@ -22,18 +16,10 @@ cd /usr/src/arvados/sdk/cli
 run_bundler
 bundler_binstubs
 
-export PYCMD=python3
-
-pip_install wheel
-
-cd /usr/src/arvados/sdk/python
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/services/fuse
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/sdk/cwl
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
+python_srcdir="$(mktemp --directory --tmpdir pysrc.XXXXXXXX)"
+trap 'rm -rf "$python_srcdir"' EXIT INT TERM QUIT
+for subdir in sdk/python services/fuse sdk/cwl; do
+    env -C "/usr/src/arvados/$subdir" \
+        /opt/arvados-py/bin/python3 setup.py sdist --dist-dir="$python_srcdir"
+done
+/opt/arvados-py/bin/pip install "$python_srcdir"/*

diff --git a/tools/arvbox/lib/arvbox/docker/yml_override.py b/tools/arvbox/lib/arvbox/docker/yml_override.py
index 7f35ac1d686984fbbc51101f8aa1a508e8ae28e0..deea83f909b27ddc715313d40a7e8292298e53b5 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/yml_override.py
+++ b/tools/arvbox/lib/arvbox/docker/yml_override.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/opt/arvados-py/bin/python3
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: AGPL-3.0