This is preparation to let us build arvbox on Debian 12, which has
adopted PEP 668 and doesn't permit installing packages outside a
virtualenv anymore.
This does move pdoc installation out of the doc service and into the
Dockerfile. The doc code currently doesn't have any way to version this
dependency, and it rarely changes in development, so this seems fine as
a code simplification.
All the code that tries to "pip install from cache only, then use the
network" is gone because as best I can tell it's already non-functional:
the pip cache isn't structured in the way that it assumes
anymore. Generally pip looks at what's installed in the virtualenv and
only installs what it *must* to satisfy dependencies, so this seems
fine.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>
ENV DEBIAN_FRONTEND noninteractive
# The arvbox-specific dependencies are
ENV DEBIAN_FRONTEND noninteractive
# The arvbox-specific dependencies are
-# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less
+# gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less
RUN apt-get update && \
apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
RUN apt-get update && \
apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
- gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat-openbsd less vim-tiny && \
+ gnupg2 runit python3-dev python3-venv shellinabox netcat-openbsd less vim-tiny && \
apt-get clean
ENV GOPATH /var/lib/gopath
apt-get clean
ENV GOPATH /var/lib/gopath
RUN $GOPATH/bin/arvados-server --version
RUN $GOPATH/bin/arvados-server install -type test
RUN $GOPATH/bin/arvados-server --version
RUN $GOPATH/bin/arvados-server install -type test
+# Set up a virtualenv for all Python tools in arvbox.
+# This is used mainly by the `sdk` service, but `doc` and internal scripts
+# also rely on it.
+# 1. Install wheel just to modernize the virtualenv.
+# 2. Install setuptools as an sdk build dependency; PyYAML for all tests
+# and yml_override;py; and pdoc for the doc service.
+# Everything else is installed by the sdk service on boot.
+RUN python3 -m venv /opt/arvados-py \
+ && /opt/arvados-py/bin/pip install --no-cache-dir wheel \
+ && /opt/arvados-py/bin/pip install --no-cache-dir setuptools PyYAML pdoc \
+ && ln -s /opt/arvados-py/bin/pdoc /usr/local/bin/
+
RUN /etc/init.d/postgresql start && \
su postgres -c 'dropuser arvados' && \
su postgres -c 'createuser -s arvbox' && \
RUN /etc/init.d/postgresql start && \
su postgres -c 'dropuser arvados' && \
su postgres -c 'createuser -s arvbox' && \
export BUNDLER_VERSION=2.4.22
export DEBIAN_FRONTEND=noninteractive
export BUNDLER_VERSION=2.4.22
export DEBIAN_FRONTEND=noninteractive
-export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=${PATH}:/usr/local/go/bin:/var/lib/arvados/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
fi
flock $GEMLOCK $BUNDLER binstubs --all
}
fi
flock $GEMLOCK $BUNDLER binstubs --all
}
-
-PYCMD=""
-pip_install() {
- pushd /var/lib/pip
- for p in $(ls http*.tar.gz) $(ls http*.tar.bz2) $(ls http*.whl) $(ls http*.zip) ; do
- if test -f $p ; then
- ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
- fi
- done
- popd
-
- if [ "$PYCMD" = "python3" ]; then
- if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
- pip3 install --prefix /usr/local $1
- fi
- else
- if ! pip install --no-index --find-links /var/lib/pip $1 ; then
- pip install $1
- fi
- fi
-}
mkdir -p $ARVADOS_CONTAINER_PATH/git \
/var/lib/passenger /var/lib/gopath \
/var/lib/pip /var/lib/npm
mkdir -p $ARVADOS_CONTAINER_PATH/git \
/var/lib/passenger /var/lib/gopath \
/var/lib/pip /var/lib/npm
+ /opt/arvados-py/bin/pip config --site set global.cache-dir /var/lib/pip
if test -z "$ARVBOX_HOME" ; then
ARVBOX_HOME=$ARVADOS_CONTAINER_PATH
if test -z "$ARVBOX_HOME" ; then
ARVBOX_HOME=$ARVADOS_CONTAINER_PATH
useradd --groups docker crunch
if [[ "$1" != --no-chown ]] ; then
useradd --groups docker crunch
if [[ "$1" != --no-chown ]] ; then
- chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH \
+ chown arvbox:arvbox -R /usr/local /opt/arvados-py $ARVADOS_CONTAINER_PATH \
/var/lib/passenger /var/lib/postgresql \
/var/lib/nginx /var/log/nginx /etc/ssl/private \
/var/lib/gopath /var/lib/pip /var/lib/npm
/var/lib/passenger /var/lib/postgresql \
/var/lib/nginx /var/log/nginx /etc/ssl/private \
/var/lib/gopath /var/lib/pip /var/lib/npm
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
-export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/usr/src/arvados/sdk/cli/binstubs
+export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin:/opt/arvados-py/bin:/usr/src/arvados/sdk/cli/binstubs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
export npm_config_cache=/var/lib/npm
export npm_config_cache_min=Infinity
export R_LIBS=/var/lib/Rlibs
+#!/opt/arvados-py/bin/python3
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
run_bundler --without=development
# Generating the Python and R docs is expensive, so for development if the file
run_bundler --without=development
# Generating the Python and R docs is expensive, so for development if the file
-# "no-sdk" exists then skip the Python and R stuff.
+# "no-sdk" exists then skip installing R stuff.
if [[ ! -f /usr/src/arvados/doc/no-sdk ]] ; then
cd /usr/src/arvados/sdk/R
R --quiet --vanilla --file=install_deps.R
if [[ ! -f /usr/src/arvados/doc/no-sdk ]] ; then
cd /usr/src/arvados/sdk/R
R --quiet --vanilla --file=install_deps.R
-
- export PYCMD=python3
- pip_install pdoc
fi
if test "$1" = "--only-deps" ; then
fi
if test "$1" = "--only-deps" ; then
. /usr/local/lib/arvbox/common.sh
. /usr/local/lib/arvbox/common.sh
-mkdir -p ~/.pip /var/lib/pip
-cat > ~/.pip/pip.conf <<EOF
-[global]
-download_cache = /var/lib/pip
-EOF
-
cd /usr/src/arvados/sdk/ruby
run_bundler
bundler_binstubs
cd /usr/src/arvados/sdk/ruby
run_bundler
bundler_binstubs
run_bundler
bundler_binstubs
run_bundler
bundler_binstubs
-export PYCMD=python3
-
-pip_install wheel
-
-cd /usr/src/arvados/sdk/python
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/services/fuse
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
-
-cd /usr/src/arvados/sdk/cwl
-$PYCMD setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
+python_srcdir="$(mktemp --directory --tmpdir pysrc.XXXXXXXX)"
+trap 'rm -rf "$python_srcdir"' EXIT INT TERM QUIT
+for subdir in sdk/python services/fuse sdk/cwl; do
+ env -C "/usr/src/arvados/$subdir" \
+ /opt/arvados-py/bin/python3 setup.py sdist --dist-dir="$python_srcdir"
+done
+/opt/arvados-py/bin/pip install "$python_srcdir"/*
+#!/opt/arvados-py/bin/python3
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0