21367/21230: Set up dev-jobs with virtualenv

The functional goal here is to do all pip installs inside the Dockerfile
inside a virtualenv to avoid the global pip install issues.

Changes that fall out of that:

* Installing python3-venv is all we need to go back to the -slim Debian
  image.
* Stop installing other Python packages we'll just install inside the
  virtualenv anyway.
* Rather than having separate arguments and code blocks for every
  package we might want to install, just set up a dedicated build
  context for the Docker image with all the packages we want to install,
  and install them all unconditionally during the image build. This is
  much less code and hopefully easier to follow.

Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith@curii.com>

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index dfa437fdfcced0354c063e8e04051e8d12008a4f..a0ea05383ae5149d2ad88187fde322bd27e4c90b 100755 (executable)
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -17,7 +17,6 @@ WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
 SALAD=path             (optional) Path to schema_salad git repository.
 CWL_UTILS=path         (optional) Path to cwl-utils git repository.
-PYCMD=pythonexec       (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
 
 EOF
 
@@ -28,64 +27,31 @@ if [[ -z "$WORKSPACE" ]] ; then
     echo "Using WORKSPACE $WORKSPACE"
 fi
 
-cd "$WORKSPACE"
-
-py=python3
-pipcmd=pip
-if [[ -n "$PYCMD" ]] ; then
-    py="$PYCMD"
-fi
-if [[ $py = python3 ]] ; then
-    pipcmd=pip3
-fi
-
-(cd sdk/python && python3 setup.py sdist)
-sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
-
-(cd sdk/cwl && python3 setup.py sdist)
-runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
-
-rm -rf sdk/cwl/salad_dist
-mkdir -p sdk/cwl/salad_dist
-if [[ -n "$SALAD" ]] ; then
-    (cd "$SALAD" && python3 setup.py sdist)
-    salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
-    cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
-fi
-
-rm -rf sdk/cwl/cwltool_dist
-mkdir -p sdk/cwl/cwltool_dist
-if [[ -n "$CWLTOOL" ]] ; then
-    (cd "$CWLTOOL" && python3 setup.py sdist)
-    cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
-    cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
-fi
-
-rm -rf sdk/cwl/cwlutils_dist
-mkdir -p sdk/cwl/cwlutils_dist
-if [[ -n "$CWL_UTILS" ]] ; then
-    (cd "$CWL_UTILS" && python3 setup.py sdist)
-    cwlutils=$(cd "$CWL_UTILS/dist" && ls -t cwl-utils-*.tar.gz | head -n1)
-    cp "$CWL_UTILS/dist/$cwlutils" $WORKSPACE/sdk/cwl/cwlutils_dist
-fi
+context_dir="$(mktemp --directory --tmpdir dev-jobs.XXXXXXXX)"
+trap 'rm -rf "$context_dir"' EXIT INT TERM QUIT
+ts_path="$context_dir/.timestamp"
+
+for src_dir in "$WORKSPACE/sdk/python" "${CWLTOOL:-}" "${CWL_UTILS:-}" "${SALAD:-}" "$WORKSPACE/sdk/cwl"; do
+    if [[ -z "$src_dir" ]]; then
+        continue
+    fi
+    touch "$ts_path"
+    env -C "$src_dir" python3 setup.py sdist
+    find "$src_dir/dist/" -maxdepth 1 -type f -cnewer "$ts_path" -print0 \
+        | xargs -0 cp --target="$context_dir/"
+done
+rm "$ts_path"
 
+cd "$WORKSPACE"
 . build/run-library.sh
-
 # This defines python_sdk_version and cwl_runner_version with python-style
 # package suffixes (.dev/rc)
 calculate_python_sdk_cwl_package_versions
 
 set -x
 docker build --no-cache \
-       --build-arg sdk=$sdk \
-       --build-arg runner=$runner \
-       --build-arg salad=$salad \
-       --build-arg cwltool=$cwltool \
-       --build-arg pythoncmd=$py \
-       --build-arg pipcmd=$pipcmd \
-       --build-arg cwlutils=$cwlutils \
        -f "$WORKSPACE/sdk/dev-jobs.dockerfile" \
        -t arvados/jobs:$cwl_runner_version \
-       "$WORKSPACE/sdk"
+       "$context_dir"
 
 arv-keepdocker arvados/jobs $cwl_runner_version

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 656572eb4fd1714485e7b4ac8b9c9bc9fd9f198e..016163307f20faad99061c093cb596eed7b21577 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -9,47 +9,20 @@
 # version.
 #
 # Use arvados/build/build-dev-docker-jobs-image.sh to build.
-#
-# (This dockerfile file must be located in the arvados/sdk/ directory because
-#  of the docker build root.)
 
-FROM debian:bullseye
+FROM debian:bullseye-slim
 MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG pythoncmd=python3
-ARG pipcmd=pip3
-
-RUN apt-get update -q && apt-get install -qy --no-install-recommends \
-    git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
-    libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential ${pythoncmd}-setuptools
-
-ARG sdk
-ARG runner
-ARG salad
-ARG cwlutils
-ARG cwltool
-
-ADD python/dist/$sdk /tmp/
-ADD cwl/salad_dist/$salad /tmp/
-ADD cwl/cwltool_dist/$cwltool /tmp/
-ADD cwl/cwlutils_dist/$cwlutils /tmp/
-ADD cwl/dist/$runner /tmp/
-
-RUN $pipcmd install wheel
-RUN cd /tmp/arvados-python-client-* && $pipcmd install .
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwl-utils-* ; then cd /tmp/cwl-utils-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
-RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
+RUN DEBIAN_FRONTEND=noninteractive apt-get update -q && apt-get install -qy --no-install-recommends \
+    git python3-dev python3-venv libcurl4-gnutls-dev libgnutls28-dev nodejs build-essential
 
-# Sometimes Python dependencies install successfully but don't
-# actually work.  So run arvados-cwl-runner here to catch fun
-# dependency errors like pkg_resources.DistributionNotFound.
-RUN arvados-cwl-runner --version
+ADD * /usr/local/src/
+RUN python3 -m venv /opt/arvados-py
+ENV PATH=/opt/arvados-py/bin:/usr/local/bin:/usr/bin:/bin
+RUN python3 -m pip install --no-cache-dir wheel
+# Run a-c-r afterward to check for a successful install.
+RUN python3 -m pip install --no-cache-dir /usr/local/src/* && arvados-cwl-runner --version
 
-# Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \
       --gecos 'Crunch execution user' crunch && \
     /usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job