18766: Merge branch 'main' into 18766-fix-ubuntu1804-python3-path

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/build/package-testing/test-package-python3-arvados-python-client.sh b/build/package-testing/test-package-python3-arvados-python-client.sh
index 69f728c10e5c335967fac801c9f131726bce18a6..1e294fe0a8be0e4b67511e7f4648116f822f5562 100755 (executable)
--- a/build/package-testing/test-package-python3-arvados-python-client.sh
+++ b/build/package-testing/test-package-python3-arvados-python-client.sh
@@ -7,7 +7,9 @@ set -e
 
 arv-put --version >/dev/null
 
-/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+PYTHON=`ls /usr/share/python3*/dist/python3-arvados-python-client/bin/python3 |head -n1`
+
+$PYTHON << EOF
 import arvados
 print("Successfully imported arvados")
 EOF

diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh
index e36c4e88c0d0bf146ac88a9d12587ef03b9a960e..aa4acb6a2bf7817f933f9bdb85b74789c91b154b 100755 (executable)
--- a/build/run-build-test-packages-one-target.sh
+++ b/build/run-build-test-packages-one-target.sh
@@ -15,6 +15,11 @@ Syntax:
     Build only a specific package (or ONLY_BUILD from environment)
 --arch <arch>
     Build a specific architecture (or ARCH from environment, defaults to native architecture)
+--force-build
+    Build even if the package exists upstream or if it has already been
+    built locally
+--force-test
+    Test even if there is no new untested package
 --upload
     If the build and test steps are successful, upload the packages
     to a remote apt repository (default: false)
@@ -48,7 +53,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
 fi
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,debug,upload,rc,target:,only-build:,arch:,build-version: \
+    help,debug,upload,rc,target:,force-test,only-build:,force-build,arch:,build-version: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -72,6 +77,12 @@ while [ $# -gt 0 ]; do
         --target)
             TARGET="$2"; shift
             ;;
+        --force-test)
+            FORCE_TEST=1
+            ;;
+        --force-build)
+            FORCE_BUILD=1
+            ;;
         --only-build)
             ONLY_BUILD="$2"; shift
             ;;
@@ -107,6 +118,14 @@ if [[ -n "$ONLY_BUILD" ]]; then
   build_args+=(--only-build "$ONLY_BUILD")
 fi
 
+if [[ -n "$FORCE_BUILD" ]]; then
+  build_args+=(--force-build)
+fi
+
+if [[ -n "$FORCE_TEST" ]]; then
+  build_args+=(--force-test)
+fi
+
 if [[ -n "$ARCH" ]]; then
   build_args+=(--arch "$ARCH")
 fi

diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 7d59eea8b3c795b3f736155216c916f535c29689..abaa190c8c781e26529fe795be0c06a303a8ace0 100644 (file)
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -32,6 +32,12 @@ h2(#main). development main (as of 2022-03-08)
 
 "previous: Upgrading to 2.3.0":#v2_3_0
 
+h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
+
+Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
+
+This means that they are now installed under @/usr/share/python3.8@ (before, the path was @/usr/share/python3@). If you rely on the @python3@ executable from the packages (e.g. to load a virtualenv), you may need to update the path to that executable.
+
 h3. Minimum supported Ruby version is now 2.6
 
 The minimum supported Ruby version is now 2.6.  If you are running Arvados on Debian 10 or Ubuntu 18.04, you may need to switch to using RVM or upgrade your OS.  See "Install Ruby and Bundler":../install/ruby.html for more information.

diff --git a/doc/sdk/python/sdk-python.html.textile.liquid b/doc/sdk/python/sdk-python.html.textile.liquid
index 435f70e7bfda47efa06ccebe0bdf2263a8a20f13..56f0328042cdc63059a82591521a26643d6dc4c8 100644 (file)
--- a/doc/sdk/python/sdk-python.html.textile.liquid
+++ b/doc/sdk/python/sdk-python.html.textile.liquid
@@ -65,6 +65,10 @@ Type "help", "copyright", "credits" or "license" for more information.
 
 If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
 
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
+
 <notextile>
 <pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
 (python-arvados-python-client) ~$ <code class="userinput">python</code>

diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 8da58a682d45368953ff473b0eaadd3ea9f63d5f..1b75e13420bce8bf77b3d4942705ce726e5a8e6e 100644 (file)
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -26,8 +26,8 @@ RUN apt-get update -q
 RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version
 
 # use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python
-RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
+RUN PYTHON=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python|head -n1` && rm -f /usr/bin/python && ln -s $PYTHON /usr/bin/python
+RUN PYTHON3=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python3|head -n1` && rm -f /usr/bin/python3 && ln -s $PYTHON3 /usr/bin/python3
 
 # Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \