Merge branch '17040-slow-query' refs #17040
authorPeter Amstutz <peter.amstutz@curii.com>
Tue, 3 Nov 2020 18:29:21 +0000 (13:29 -0500)
committerPeter Amstutz <peter.amstutz@curii.com>
Tue, 3 Nov 2020 18:29:21 +0000 (13:29 -0500)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

60 files changed:
build/README
build/build-dev-docker-jobs-image.sh
build/package-build-dockerfiles/Makefile
build/package-build-dockerfiles/centos7/Dockerfile
build/package-build-dockerfiles/debian9/Dockerfile [deleted file]
build/package-test-dockerfiles/Makefile
build/package-test-dockerfiles/debian9/Dockerfile [deleted file]
build/package-testing/test-package-rh-python36-python-arvados-python-client.sh
build/rails-package-scripts/README.md
build/rails-package-scripts/arvados-sso-server.sh [deleted file]
build/rails-package-scripts/postinst.sh
build/run-build-docker-images.sh
build/run-build-packages-one-target.sh
build/run-build-packages-sso.sh [deleted file]
build/run-build-packages.sh
build/run-library.sh
doc/_includes/_install_ruby_and_bundler.liquid
doc/admin/upgrading.html.textile.liquid
doc/install/crunch2-slurm/install-dispatch.html.textile.liquid
doc/install/install-manual-prerequisites.html.textile.liquid
doc/install/install-postgresql.html.textile.liquid
doc/install/packages.html.textile.liquid
doc/sdk/cli/install.html.textile.liquid
doc/sdk/python/arvados-cwl-runner.html.textile.liquid
doc/sdk/python/arvados-fuse.html.textile.liquid
doc/sdk/python/sdk-python.html.textile.liquid
doc/sdk/ruby/index.html.textile.liquid
doc/user/composer/composer.html.textile.liquid
doc/user/cwl/cwl-extensions.html.textile.liquid
docker/jobs/Dockerfile
lib/controller/fed_collections.go
lib/controller/fed_containers.go
lib/controller/fed_generic.go
lib/controller/localdb/login_oidc.go
lib/crunchrun/background.go
lib/service/cmd.go
sdk/cli/arvados-cli.gemspec
sdk/cwl/fpm-info.sh
sdk/cwl/tests/test_submit.py
sdk/cwl/tests/tool/submit_tool.cwl
sdk/cwl/tests/tool/tool_with_sf.cwl
sdk/cwl/tests/wf/16169-step.cwl
sdk/cwl/tests/wf/expect_arvworkflow.cwl
sdk/cwl/tests/wf/expect_packed.cwl
sdk/cwl/tests/wf/expect_upload_packed.cwl
sdk/cwl/tests/wf/secret_wf.cwl
sdk/cwl/tests/wf/submit_wf_packed.cwl
sdk/dev-jobs.dockerfile
sdk/go/arvados/client.go
sdk/go/arvados/link.go
sdk/go/keepclient/keepclient.go
sdk/go/keepclient/support.go
sdk/python/README.rst
sdk/ruby/arvados.gemspec
services/crunch-dispatch-local/crunch-dispatch-local.service [new file with mode: 0644]
services/crunch-dispatch-local/fpm-info.sh [new file with mode: 0644]
services/dockercleaner/fpm-info.sh
services/keep-web/s3aws_test.go
services/keepproxy/keepproxy.go
services/login-sync/arvados-login-sync.gemspec

index 4c67839a1006693f3d70f8adaf2823e4fa4f11e5..54d5ea404cdef205d3cd08ffabcdd6d78d775eb7 100644 (file)
@@ -16,8 +16,6 @@ run-build-packages.sh                    Actually build packages.  Intended to r
                                          inside Docker container with proper
                                          build environment.
 
-run-build-packages-sso.sh                Build single-sign-on server packages.
-
 run-build-packages-python-and-ruby.sh    Build Python and Ruby packages suitable
                                          for upload to PyPi and Rubygems.
 
@@ -31,4 +29,4 @@ build-dev-docker-jobs-image.sh           Build developer arvados/jobs Docker ima
 
 run-library.sh                           A library of functions shared by the
                                          various scripts in this
-                                         directory.
\ No newline at end of file
+                                         directory.
index 3fb1da7990929e859b9b7da21521702f13b916b0..0e570d5f31838037160f5797f80e1fc0cc7048e4 100755 (executable)
@@ -16,7 +16,7 @@ Syntax:
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
 SALAD=path             (optional) Path to schema_salad git repository.
-PYCMD=pythonexec       (optional) Specify the python executable to use in the docker image. Defaults to "python3".
+PYCMD=pythonexec       (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
 
 EOF
 
@@ -45,16 +45,16 @@ if [[ $py = python3 ]] ; then
     pipcmd=pip3
 fi
 
-(cd sdk/python && python setup.py sdist)
+(cd sdk/python && python3 setup.py sdist)
 sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
 
-(cd sdk/cwl && python setup.py sdist)
+(cd sdk/cwl && python3 setup.py sdist)
 runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
 
 rm -rf sdk/cwl/salad_dist
 mkdir -p sdk/cwl/salad_dist
 if [[ -n "$SALAD" ]] ; then
-    (cd "$SALAD" && python setup.py sdist)
+    (cd "$SALAD" && python3 setup.py sdist)
     salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
     cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
 fi
@@ -62,7 +62,7 @@ fi
 rm -rf sdk/cwl/cwltool_dist
 mkdir -p sdk/cwl/cwltool_dist
 if [[ -n "$CWLTOOL" ]] ; then
-    (cd "$CWLTOOL" && python setup.py sdist)
+    (cd "$CWLTOOL" && python3 setup.py sdist)
     cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
     cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
 fi
index e116cb7fc3ffb2688abcf5a5c6fe10d6e32d4718..4763b7a2930902f5745de65bd3b1fbcabb5b2f8d 100644 (file)
@@ -2,21 +2,16 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian9/generated debian10/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian10/generated ubuntu1604/generated ubuntu1804/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
        cp -f -rlt centos7/generated common-generated/*
 
-debian9/generated: common-generated-all
-       test -d debian9/generated || mkdir debian9/generated
-       cp -f -rlt debian9/generated common-generated/*
-
 debian10/generated: common-generated-all
        test -d debian10/generated || mkdir debian10/generated
        cp -f -rlt debian10/generated common-generated/*
 
-
 ubuntu1604/generated: common-generated-all
        test -d ubuntu1604/generated || mkdir ubuntu1604/generated
        cp -f -rlt ubuntu1604/generated common-generated/*
index 5d204464cff89c27b0e21158fb42bbb77adc12cc..3c742d3b259c12707ae3dacbeafbd3055875ec62 100644 (file)
@@ -40,17 +40,15 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
 
 # Need to "touch" RPM database to workaround bug in interaction between
 # overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install rh-python36
-RUN scl enable rh-python36 "easy_install-3.6 pip"
+RUN touch /var/lib/rpm/* && yum -q -y install python3 python3-pip python3-devel
 
-# Add epel, we need it for the python-pam dependency
-#RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-#RUN rpm -ivh epel-release-latest-7.noarch.rpm
+# Install virtualenv
+RUN /usr/bin/pip3 install 'virtualenv<20'
 
 RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
 
 # The version of setuptools that comes with CentOS is way too old
-RUN scl enable rh-python36 "easy_install-3.6 pip install 'setuptools<45'"
+RUN pip3 install 'setuptools<45'
 
 ENV WORKSPACE /arvados
-CMD ["scl", "enable", "rh-python36", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "centos7"]
diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile
deleted file mode 100644 (file)
index 5294997..0000000
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-## dont use debian:9 here since the word 'stretch' is used for rvm precompiled binaries
-FROM debian:stretch
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev
-
-# Install virtualenv
-RUN /usr/bin/pip3 install 'virtualenv<20'
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
-    gpg --import --no-tty /tmp/pkuczynski.asc && \
-    curl -L https://get.rvm.io | bash -s stable && \
-    /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
-    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2 && \
-    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-
-# Install golang binary
-ADD generated/go1.13.4.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-
-ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
index 4bc13b0f34a205db65bc8afbaa384454f39cb8c6..f7a68c28abe38dcb7568480e051b16a54075bb9e 100644 (file)
@@ -2,16 +2,12 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian9/generated  debian10/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian10/generated ubuntu1604/generated ubuntu1804/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
        cp -f -rlt centos7/generated common-generated/*
 
-debian9/generated: common-generated-all
-       test -d debian9/generated || mkdir debian9/generated
-       cp -f -rlt debian9/generated common-generated/*
-
 debian10/generated: common-generated-all
        test -d debian10/generated || mkdir debian10/generated
        cp -f -rlt debian10/generated common-generated/*
diff --git a/build/package-test-dockerfiles/debian9/Dockerfile b/build/package-test-dockerfiles/debian9/Dockerfile
deleted file mode 100644 (file)
index 423a9e7..0000000
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM debian:stretch
-MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends curl ca-certificates gpg procps
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
-    gpg --import --no-tty /tmp/pkuczynski.asc && \
-    curl -L https://get.rvm.io | bash -s stable && \
-    /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
-    /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.0.2
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian9/ /" >>/etc/apt/sources.list
index 1a692565601e45d9464c70ca5a444c2c78375c2e..914974d0894ef726019d8de17275cc9945b86332 100755 (executable)
@@ -7,7 +7,7 @@ set -e
 
 arv-put --version
 
-/usr/share/python3/dist/rh-python36-python-arvados-python-client/bin/python3 << EOF
+/usr/bin/python3 << EOF
 import arvados
 print("Successfully imported arvados")
 EOF
index 2930957b942affd86326248e6e0e4a3efb3166f9..35549d9cd3b8673c0ed13fbf23386bdab6798014 100644 (file)
@@ -14,5 +14,5 @@ postinst.sh lets the early parts define a few hooks to control behavior:
 
 * After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles.  By default this is a noop function (in step2.sh).
 * Before it restarts nginx, it calls setup_before_nginx_restart.  By default this is a noop function (in step2.sh).  API server defines this to set up the internal git repository, if necessary.
-* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database.  API server uses db:structure:load.  SSO server uses db:schema:load.  Workbench doesn't set this, which causes the postinst to skip all database work.
-* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task.  SSO clears this flag (it doesn't have that task code).
+* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database.  API server uses db:structure:load.  Workbench doesn't set this, which causes the postinst to skip all database work.
+* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task.
diff --git a/build/rails-package-scripts/arvados-sso-server.sh b/build/rails-package-scripts/arvados-sso-server.sh
deleted file mode 100644 (file)
index e88da0d..0000000
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This file declares variables common to all scripts for one Rails package.
-
-PACKAGE_NAME=arvados-sso-server
-INSTALL_PATH=/var/www/arvados-sso
-CONFIG_PATH=/etc/arvados/sso
-DOC_URL="https://doc.arvados.org/v2.0/install/install-sso.html#configure"
-RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
-RAILSPKG_SUPPORTS_CONFIG_CHECK=0
index 357c1006431452385cc3a60c028e396452b1ffc6..3eb2d2c5e0c2f58d2099f131c0b3ecd0d8e3078b 100644 (file)
@@ -256,9 +256,4 @@ elif [ "$1" = "0" ] || [ "$1" = "1" ] || [ "$1" = "2" ]; then
   configure_version
 fi
 
-if printf '%s\n' "$CONFIG_PATH" | grep -Fqe "sso"; then
-       report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
-       report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
-else
-       report_not_ready "$APPLICATION_READY" "/etc/arvados/config.yml"
-fi
+report_not_ready "$APPLICATION_READY" "/etc/arvados/config.yml"
index 89818e14644fb4d4886da58ddf4e1d092efc6a23..8cff14b71e2934607c6794a5b00a461dac80338f 100755 (executable)
@@ -9,6 +9,7 @@ function usage {
     echo >&2
     echo >&2 "$0 options:"
     echo >&2 "  -t, --tags [csv_tags]         comma separated tags"
+    echo >&2 "  -i, --images [dev,demo]       Choose which images to build (default: dev and demo)"
     echo >&2 "  -u, --upload                  Upload the images (docker push)"
     echo >&2 "  -h, --help                    Display this help and exit"
     echo >&2
@@ -16,10 +17,11 @@ function usage {
 }
 
 upload=false
+images=dev,demo
 
 # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hut: \
-    --long help,upload,tags: \
+TEMP=`getopt -o hut:i: \
+    --long help,upload,tags:,images: \
     -n "$0" -- "$@"`
 
 if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -33,6 +35,19 @@ do
             upload=true
             shift
             ;;
+        -i | --images)
+            case "$2" in
+                "")
+                  echo "ERROR: --images needs a parameter";
+                  usage;
+                  exit 1
+                  ;;
+                *)
+                  images=$2;
+                  shift 2
+                  ;;
+            esac
+            ;;
         -t | --tags)
             case "$2" in
                 "")
@@ -67,6 +82,9 @@ title () {
 }
 
 docker_push () {
+    # docker always creates a local 'latest' tag, and we don't want to push that
+    # tag in every case. Remove it.
+    docker rmi $1:latest
     if [[ ! -z "$tags" ]]
     then
         for tag in $( echo $tags|tr "," " " )
@@ -128,37 +146,40 @@ timer_reset
 # clean up the docker build environment
 cd "$WORKSPACE"
 
-title "Starting arvbox build localdemo"
+if [[ "$images" =~ demo ]]; then
+  title "Starting arvbox build localdemo"
 
-tools/arvbox/bin/arvbox build localdemo
-ECODE=$?
+  tools/arvbox/bin/arvbox build localdemo
+  ECODE=$?
 
-if [[ "$ECODE" != "0" ]]; then
-    title "!!!!!! docker BUILD FAILED !!!!!!"
-    EXITCODE=$(($EXITCODE + $ECODE))
+  if [[ "$ECODE" != "0" ]]; then
+      title "!!!!!! docker BUILD FAILED !!!!!!"
+      EXITCODE=$(($EXITCODE + $ECODE))
+  fi
 fi
 
-title "Starting arvbox build dev"
+if [[ "$images" =~ dev ]]; then
+  title "Starting arvbox build dev"
 
-tools/arvbox/bin/arvbox build dev
+  tools/arvbox/bin/arvbox build dev
 
-ECODE=$?
+  ECODE=$?
 
-if [[ "$ECODE" != "0" ]]; then
-    title "!!!!!! docker BUILD FAILED !!!!!!"
-    EXITCODE=$(($EXITCODE + $ECODE))
+  if [[ "$ECODE" != "0" ]]; then
+      title "!!!!!! docker BUILD FAILED !!!!!!"
+      EXITCODE=$(($EXITCODE + $ECODE))
+  fi
 fi
 
 title "docker build complete (`timer`)"
 
-title "uploading images"
-
-timer_reset
-
 if [[ "$EXITCODE" != "0" ]]; then
     title "upload arvados images SKIPPED because build failed"
 else
     if [[ $upload == true ]]; then
+        title "uploading images"
+        timer_reset
+
         ## 20150526 nico -- *sometimes* dockerhub needs re-login
         ## even though credentials are already in .dockercfg
         docker login -u arvados
index d0a79ad3dfa2fdf04cab380f321602fac66df618..72f814836b3ce36b1a56f0c2221c9d9b50b8c485 100755 (executable)
@@ -217,22 +217,12 @@ if test -z "$packages" ; then
         keep-block-check
         keep-web
         libarvados-perl
-        libpam-arvados-go"
-    if [[ "$TARGET" =~ "centos" ]]; then
-      packages="$packages
-        rh-python36-python-cwltest
-        rh-python36-python-arvados-fuse
-        rh-python36-python-arvados-python-client
-        rh-python36-python-arvados-cwl-runner
-        rh-python36-python-crunchstat-summary"
-    else
-      packages="$packages
+        libpam-arvados-go
         python3-cwltest
         python3-arvados-fuse
         python3-arvados-python-client
         python3-arvados-cwl-runner
         python3-crunchstat-summary"
-    fi
 fi
 
 FINAL_EXITCODE=0
diff --git a/build/run-build-packages-sso.sh b/build/run-build-packages-sso.sh
deleted file mode 100755 (executable)
index d8d9b98..0000000
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-JENKINS_DIR=$(dirname $(readlink -e "$0"))
-. "$JENKINS_DIR/run-library.sh"
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Build Arvados SSO server package
-
-Syntax:
-        WORKSPACE=/path/to/arvados-sso $(basename $0) [options]
-
-Options:
-
---debug
-    Output debug information (default: false)
---target
-    Distribution to build packages for (default: debian10)
-
-WORKSPACE=path         Path to the Arvados SSO source tree to build packages from
-
-EOF
-
-EXITCODE=0
-DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian10
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,build-bundle-packages,debug,target: \
-    -- "" "$@")
-if [ $? -ne 0 ]; then
-    exit 1
-fi
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --help)
-            echo >&2 "$helpmessage"
-            echo >&2
-            exit 1
-            ;;
-        --target)
-            TARGET="$2"; shift
-            ;;
-        --debug)
-            DEBUG=1
-            ;;
-        --test-packages)
-            test_packages=1
-            ;;
-        --)
-            if [ $# -gt 1 ]; then
-                echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
-                exit 1
-            fi
-            ;;
-    esac
-    shift
-done
-
-STDOUT_IF_DEBUG=/dev/null
-STDERR_IF_DEBUG=/dev/null
-DASHQ_UNLESS_DEBUG=-q
-if [[ "$DEBUG" != 0 ]]; then
-    STDOUT_IF_DEBUG=/dev/stdout
-    STDERR_IF_DEBUG=/dev/stderr
-    DASHQ_UNLESS_DEBUG=
-fi
-
-case "$TARGET" in
-    debian*)
-        FORMAT=deb
-        ;;
-    ubuntu*)
-        FORMAT=deb
-        ;;
-    centos*)
-        FORMAT=rpm
-        ;;
-    *)
-        echo -e "$0: Unknown target '$TARGET'.\n" >&2
-        exit 1
-        ;;
-esac
-
-if ! [[ -n "$WORKSPACE" ]]; then
-  echo >&2 "$helpmessage"
-  echo >&2
-  echo >&2 "Error: WORKSPACE environment variable not set"
-  echo >&2
-  exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
-  echo >&2 "$helpmessage"
-  echo >&2
-  echo >&2 "Error: $WORKSPACE is not a directory"
-  echo >&2
-  exit 1
-fi
-
-# Test for fpm
-fpm --version >/dev/null 2>&1
-
-if [[ "$?" != 0 ]]; then
-    echo >&2 "$helpmessage"
-    echo >&2
-    echo >&2 "Error: fpm not found"
-    echo >&2
-    exit 1
-fi
-
-RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
-RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
-if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
-    # error; for some reason, the path is not accessible
-    # to the script (e.g. permissions re-evaled after suid)
-    exit 1  # fail
-fi
-
-debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
-debug_echo "Workspace is $WORKSPACE"
-
-if [[ -f /etc/profile.d/rvm.sh ]]; then
-    source /etc/profile.d/rvm.sh
-    GEM="rvm-exec default gem"
-else
-    GEM=gem
-fi
-
-# Make all files world-readable -- jenkins runs with umask 027, and has checked
-# out our git tree here
-chmod o+r "$WORKSPACE" -R
-
-# More cleanup - make sure all executables that we'll package are 755
-# No executables in the sso server package
-#find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
-
-# Now fix our umask to something better suited to building and publishing
-# gems and packages
-umask 0022
-
-debug_echo "umask is" `umask`
-
-if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
-    mkdir -p "$WORKSPACE/packages/$TARGET"
-fi
-
-# Build the SSO server package
-handle_rails_package arvados-sso-server "$WORKSPACE" \
-                     "$WORKSPACE/LICENCE" --url="https://arvados.org" \
-                     --description="Arvados SSO server - Arvados is a free and open source platform for big data science." \
-                     --license="Expat license"
-
-exit $EXITCODE
index 0e74ac6f2570761d34cfc91d58b36d16c1fa812d..8d55e2fd9be7d3cf0f47e179b2645e8fca89d540 100755 (executable)
@@ -125,7 +125,7 @@ case "$TARGET" in
         FORMAT=rpm
         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
-        PYTHON3_PREFIX=/opt/rh/rh-python36/root/usr
+        PYTHON3_PREFIX=/usr
         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
         export PYCURL_SSL_LIBRARY=nss
         ;;
index bd4d1aeb1f65ac00d0594ea15f9ae7bba3f13063..b74ef3ae3092fd81e7bc72a5dffa212d94c16a84 100755 (executable)
@@ -352,7 +352,6 @@ test_package_presence() {
       echo "Package $full_pkgname build forced with --force-build, building"
     elif [[ "$FORMAT" == "deb" ]]; then
       declare -A dd
-      dd[debian9]=stretch
       dd[debian10]=buster
       dd[ubuntu1604]=xenial
       dd[ubuntu1804]=bionic
@@ -432,9 +431,7 @@ handle_rails_package() {
     fi
     # For some reason fpm excludes need to not start with /.
     local exclude_root="${railsdir#/}"
-    # .git and packages are for the SSO server, which is built from its
-    # repository root.
-    local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+    local -a exclude_list=(tmp log coverage Capfile\* \
                            config/deploy\* config/application.yml)
     # for arvados-workbench, we need to have the (dummy) config/database.yml in the package
     if  [[ "$pkgname" != "arvados-workbench" ]]; then
@@ -475,12 +472,7 @@ fpm_build_virtualenv () {
   case "$PACKAGE_TYPE" in
     python3)
         python=python3
-        if [[ "$FORMAT" != "rpm" ]]; then
-          pip=pip3
-        else
-          # In CentOS, we use a different mechanism to get the right version of pip
-          pip=pip
-        fi
+        pip=pip3
         PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
         ;;
   esac
index d1d33cbbe3e567a610768cbbe790560b2e953d7d..7be699d3fb4f88eee2a3f6c4b65bd901192f3358 100644 (file)
@@ -22,7 +22,7 @@ The Ruby version shipped with Centos 7 is too old.  Use "RVM":#rvm to install Ru
 
 h3. Debian and Ubuntu
 
-Debian 9 (stretch) and Ubuntu 16.04 (xenial) ship Ruby 2.3, which is not supported by Arvados.  Use "RVM":#rvm to install Ruby 2.5 or later.
+Ubuntu 16.04 (xenial) ships with Ruby 2.3, which is not supported by Arvados.  Use "RVM":#rvm to install Ruby 2.5 or later.
 
 Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5, which is supported by Arvados.
 
@@ -73,11 +73,11 @@ Finally, install Bundler:
 
 h2(#fromsource). Option 3: Install from source
 
-Install prerequisites for Debian 8:
+Install prerequisites for Debian 10:
 
 <notextile>
 <pre><code><span class="userinput">sudo apt-get install \
-    bison build-essential gettext libcurl3 libcurl3-gnutls \
+    bison build-essential gettext libcurl4 \
     libcurl4-openssl-dev libpcre3-dev libreadline-dev \
     libssl-dev libxslt1.1 zlib1g-dev
 </span></code></pre></notextile>
@@ -91,13 +91,13 @@ Install prerequisites for CentOS 7:
     make automake libtool bison sqlite-devel tar
 </span></code></pre></notextile>
 
-Install prerequisites for Ubuntu 12.04 or 14.04:
+Install prerequisites for Ubuntu 16.04:
 
 <notextile>
 <pre><code><span class="userinput">sudo apt-get install \
-    gawk g++ gcc make libc6-dev libreadline6-dev zlib1g-dev libssl-dev \
-    libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev \
-    libncurses5-dev automake libtool bison pkg-config libffi-dev curl
+    bison build-essential gettext libcurl3 \
+    libcurl3-openssl-dev libpcre3-dev libreadline-dev \
+    libssl-dev libxslt1.1 zlib1g-dev
 </span></code></pre></notextile>
 
 Build and install Ruby:
index d0a29f598be9bdd2e5187b715b16fe2305241351..e8cde5acec8fe41761e016f963a1ac8356588708 100644 (file)
@@ -35,6 +35,14 @@ TODO: extract this information based on git commit messages and generate changel
 <div class="releasenotes">
 </notextile>
 
+h2(#master). development master (as of 2020-10-28)
+
+"Upgrading from 2.1.0":#v2_1_0
+
+h3. Centos7 Python 3 dependency upgraded to python3
+
+Now that Python 3 is part of the base repository in CentOS 7, the Python 3 dependency for Centos7 Arvados packages was changed from SCL rh-python36 to python3.
+
 h2(#v2_1_0). v2.1.0 (2020-10-13)
 
 "Upgrading from 2.0.0":#v2_0_0
index a9689e9ac357842f3cca6dd69d0f8b9f43062a93..3996cc7930a70a44ace17a8bd55cade99876bd7c 100644 (file)
@@ -22,7 +22,7 @@ crunch-dispatch-slurm is only relevant for on premises clusters that will spool
 
 h2(#introduction). Introduction
 
-This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html .  For information on installing Slurm, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html
+This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html.  Slurm packages are available for CentOS, Debian and Ubuntu. Please see your distribution package repositories. For information on installing Slurm from source, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html
 
 The Arvados Slurm dispatcher can run on any node that can submit requests to both the Arvados API server and the Slurm controller (via @sbatch@).  It is not resource-intensive, so you can run it on the API server node.
 
index 534309a7eb8dea2577b3d16c3bacf10795b8bd08..e6f1ba8fdcdb6e562831f197ae1a262dc76b25a1 100644 (file)
@@ -30,11 +30,11 @@ table(table table-bordered table-condensed).
 |_. Distribution|_. State|_. Last supported version|
 |CentOS 7|Supported|Latest|
 |Debian 10 ("buster")|Supported|Latest|
-|Debian 9 ("stretch")|Supported|Latest|
 |Ubuntu 18.04 ("bionic")|Supported|Latest|
 |Ubuntu 16.04 ("xenial")|Supported|Latest|
-|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
+|Debian 9 ("stretch")|EOL|Latest 2.1.X release|
 |Debian 8 ("jessie")|EOL|1.4.3|
+|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
 |Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
 |Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
 |CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
index b25194a9eebd79a067719fee379849e2c7c1dfc6..60afa1e24fa51b50237c308128b708d006d71d24 100644 (file)
@@ -19,20 +19,18 @@ h3(#centos7). CentOS 7
 {% include 'note_python_sc' %}
 
 # Install PostgreSQL
-  <notextile><pre># <span class="userinput">yum install rh-postgresql95 rh-postgresql95-postgresql-contrib</span>
-~$ <span class="userinput">scl enable rh-postgresql95 bash</span></pre></notextile>
+  <notextile><pre># <span class="userinput">yum install rh-postgresql12 rh-postgresql12-postgresql-contrib</span>
+~$ <span class="userinput">scl enable rh-postgresql12 bash</span></pre></notextile>
 # Initialize the database
   <notextile><pre># <span class="userinput">postgresql-setup initdb</span></pre></notextile>
 # Configure the database to accept password connections
   <notextile><pre><code># <span class="userinput">sed -ri -e 's/^(host +all +all +(127\.0\.0\.1\/32|::1\/128) +)ident$/\1md5/' /var/lib/pgsql/data/pg_hba.conf</span></code></pre></notextile>
 # Configure the database to launch at boot and start now
-  <notextile><pre># <span class="userinput">systemctl enable --now rh-postgresql95-postgresql</span></pre></notextile>
+  <notextile><pre># <span class="userinput">systemctl enable --now rh-postgresql12-postgresql</span></pre></notextile>
 
 h3(#debian). Debian or Ubuntu
 
-Debian 8 (Jessie) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
-
-Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/
+Debian 10 (Buster) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
 
 # Install PostgreSQL
   <notextile><pre># <span class="userinput">apt-get --no-install-recommends install postgresql postgresql-contrib</span></pre></notextile>
index ed392b6667f1257362eaa38706dae2eae5fbe8dd..cb7102bb3770ebaa74fba78317446c78c7c215a1 100644 (file)
@@ -42,7 +42,6 @@ As root, add the Arvados package repository to your sources.  This command depen
 table(table table-bordered table-condensed).
 |_. OS version|_. Command|
 |Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ buster main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 |Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 |Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 
index 3c60bdfe3a9569d7287c94bb8659b3e153266241..9657d236addf3c2dd89d154ac9dd28b801cfd064 100644 (file)
@@ -17,7 +17,7 @@ h2. Prerequisites
 # "Install Ruby":../../install/ruby.html
 # "Install the Python SDK":../python/sdk-python.html
 
-The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 10 this is:
 
 <pre>
 $ apt-get install build-essential libcurl4-openssl-dev
index 9faedb8dc13437baa124cb1c8103d7f11e5eb963..1cfbd6054566f5d8ab394902d46ca1852aa2e16b 100644 (file)
@@ -32,13 +32,7 @@ Run @pip install arvados-cwl-runner@ in an appropriate installation environment,
 
 Note:
 
-The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 9 this is:
-
-<pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
-</pre>
-
-For Python 3 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
 
 <pre>
 $ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
index 0ac2d0c7e171cc32b890d8b9eb83d362ca4b7451..04dca2c849d4a5519cfbcb93c6a96bdc4dbd4dfe 100644 (file)
@@ -32,16 +32,10 @@ Run @pip install arvados_fuse@ in an appropriate installation environment, such
 
 Note:
 
-The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
 
 <pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
-</pre>
-
-For Python 3 this is:
-
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev python3-llfuse
 </pre>
 
 h3. Usage
index 2915d554d93bbed9b73da2ba96f8658b167bc88f..e132305f0fc02d26398e71130b93c181fc68e03f 100644 (file)
@@ -38,16 +38,10 @@ Run @pip install arvados-python-client@ in an appropriate installation environme
 
 Note:
 
-The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
 
 <pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev
-</pre>
-
-For Python 3 this is
-
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev
 </pre>
 
 If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip install --pre arvados-python-client@.
index 6f06722d236b80fe7853ab655af1dcbfe5c73e2a..b3b97244bad15f643b1d434163c231acc4269d9b 100644 (file)
@@ -22,7 +22,7 @@ h3. Prerequisites
 
 # "Install Ruby":../../install/ruby.html
 
-The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 10 this is:
 
 <pre>
 $ apt-get install build-essential libcurl4-openssl-dev
index 400c55b976c566427987309edaad6a628369e9f7..b0ff8247619e7d128487af2353edbb5be9dc8948 100644 (file)
@@ -48,7 +48,7 @@ h3. 6. Create a new Command Line Tool
 
 h3. 7. Set Docker image, base command, and input port for "sort" tool
 
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:10@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
 
 !(screenshot)c6.png!
 
index 505cfc4f597e394a4b79690aa1db972d84f1bce7..09a553becfbff4d65ecbc9c82467b2cf557c8640 100644 (file)
@@ -127,7 +127,7 @@ This is an optional extension field appearing on the standard @DockerRequirement
 <pre>
 requirements:
   DockerRequirement:
-    dockerPull: "debian:9"
+    dockerPull: "debian:10"
     arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
 </pre>
 
index 69ea34bc81c412f0ec21d6747db904a163f3000f..8da58a682d45368953ff473b0eaadd3ea9f63d5f 100644 (file)
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# Based on Debian Stretch
+# Based on Debian
 FROM debian:buster-slim
 MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
index c33f5b28946ab430e8532195c50c8ff8ac478506..a0a123129fdacdae34bf8b216d3e6b766a6f5889 100644 (file)
@@ -157,7 +157,7 @@ type searchRemoteClusterForPDH struct {
 func fetchRemoteCollectionByUUID(
        h *genericFederatedRequestHandler,
        effectiveMethod string,
-       clusterId *string,
+       clusterID *string,
        uuid string,
        remainder string,
        w http.ResponseWriter,
@@ -170,11 +170,11 @@ func fetchRemoteCollectionByUUID(
 
        if uuid != "" {
                // Collection UUID GET request
-               *clusterId = uuid[0:5]
-               if *clusterId != "" && *clusterId != h.handler.Cluster.ClusterID {
+               *clusterID = uuid[0:5]
+               if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID {
                        // request for remote collection by uuid
-                       resp, err := h.handler.remoteClusterRequest(*clusterId, req)
-                       newResponse, err := rewriteSignatures(*clusterId, "", resp, err)
+                       resp, err := h.handler.remoteClusterRequest(*clusterID, req)
+                       newResponse, err := rewriteSignatures(*clusterID, "", resp, err)
                        h.handler.proxy.ForwardResponse(w, newResponse, err)
                        return true
                }
@@ -186,7 +186,7 @@ func fetchRemoteCollectionByUUID(
 func fetchRemoteCollectionByPDH(
        h *genericFederatedRequestHandler,
        effectiveMethod string,
-       clusterId *string,
+       clusterID *string,
        uuid string,
        remainder string,
        w http.ResponseWriter,
index c62cea1168eb29c212ad5eefdd7a9d58dc609f8c..51f243e69e48d106bec6239c8cc8a66597dff60e 100644 (file)
@@ -19,7 +19,7 @@ import (
 func remoteContainerRequestCreate(
        h *genericFederatedRequestHandler,
        effectiveMethod string,
-       clusterId *string,
+       clusterID *string,
        uuid string,
        remainder string,
        w http.ResponseWriter,
@@ -42,7 +42,7 @@ func remoteContainerRequestCreate(
                return true
        }
 
-       if *clusterId == "" || *clusterId == h.handler.Cluster.ClusterID {
+       if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID {
                // Submitting container request to local cluster. No
                // need to set a runtime_token (rails api will create
                // one when the container runs) or do a remote cluster
@@ -117,7 +117,7 @@ func remoteContainerRequestCreate(
        req.ContentLength = int64(buf.Len())
        req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
 
-       resp, err := h.handler.remoteClusterRequest(*clusterId, req)
+       resp, err := h.handler.remoteClusterRequest(*clusterID, req)
        h.handler.proxy.ForwardResponse(w, resp, err)
        return true
 }
index 476fd97b05cd1c8a10ded9aaf43ef6b21744443c..fc2d96cc55fb5f4f0be7e46f55ee3f70445078a3 100644 (file)
@@ -20,7 +20,7 @@ import (
 type federatedRequestDelegate func(
        h *genericFederatedRequestHandler,
        effectiveMethod string,
-       clusterId *string,
+       clusterID *string,
        uuid string,
        remainder string,
        w http.ResponseWriter,
@@ -38,12 +38,12 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter,
        clusterID string, uuids []string) (rp []map[string]interface{}, kind string, err error) {
 
        found := make(map[string]bool)
-       prev_len_uuids := len(uuids) + 1
+       prevLenUuids := len(uuids) + 1
        // Loop while
        // (1) there are more uuids to query
        // (2) we're making progress - on each iteration the set of
        // uuids we are expecting for must shrink.
-       for len(uuids) > 0 && len(uuids) < prev_len_uuids {
+       for len(uuids) > 0 && len(uuids) < prevLenUuids {
                var remoteReq http.Request
                remoteReq.Header = req.Header
                remoteReq.Method = "POST"
@@ -103,7 +103,7 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter,
                                l = append(l, u)
                        }
                }
-               prev_len_uuids = len(uuids)
+               prevLenUuids = len(uuids)
                uuids = l
        }
 
@@ -111,7 +111,7 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter,
 }
 
 func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.ResponseWriter,
-       req *http.Request, clusterId *string) bool {
+       req *http.Request, clusterID *string) bool {
 
        var filters [][]interface{}
        err := json.Unmarshal([]byte(req.Form.Get("filters")), &filters)
@@ -141,17 +141,17 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response
                        if rhs, ok := filter[2].([]interface{}); ok {
                                for _, i := range rhs {
                                        if u, ok := i.(string); ok && len(u) == 27 {
-                                               *clusterId = u[0:5]
+                                               *clusterID = u[0:5]
                                                queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u)
-                                               expectCount += 1
+                                               expectCount++
                                        }
                                }
                        }
                } else if op == "=" {
                        if u, ok := filter[2].(string); ok && len(u) == 27 {
-                               *clusterId = u[0:5]
+                               *clusterID = u[0:5]
                                queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u)
-                               expectCount += 1
+                               expectCount++
                        }
                } else {
                        return false
@@ -256,10 +256,10 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response
 
 func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
        m := h.matcher.FindStringSubmatch(req.URL.Path)
-       clusterId := ""
+       clusterID := ""
 
        if len(m) > 0 && m[2] != "" {
-               clusterId = m[2]
+               clusterID = m[2]
        }
 
        // Get form parameters from URL and form body (if POST).
@@ -270,7 +270,7 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h
 
        // Check if the parameters have an explicit cluster_id
        if req.Form.Get("cluster_id") != "" {
-               clusterId = req.Form.Get("cluster_id")
+               clusterID = req.Form.Get("cluster_id")
        }
 
        // Handle the POST-as-GET special case (workaround for large
@@ -283,9 +283,9 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h
        }
 
        if effectiveMethod == "GET" &&
-               clusterId == "" &&
+               clusterID == "" &&
                req.Form.Get("filters") != "" &&
-               h.handleMultiClusterQuery(w, req, &clusterId) {
+               h.handleMultiClusterQuery(w, req, &clusterID) {
                return
        }
 
@@ -295,15 +295,15 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h
                uuid = m[1][1:]
        }
        for _, d := range h.delegates {
-               if d(h, effectiveMethod, &clusterId, uuid, m[3], w, req) {
+               if d(h, effectiveMethod, &clusterID, uuid, m[3], w, req) {
                        return
                }
        }
 
-       if clusterId == "" || clusterId == h.handler.Cluster.ClusterID {
+       if clusterID == "" || clusterID == h.handler.Cluster.ClusterID {
                h.next.ServeHTTP(w, req)
        } else {
-               resp, err := h.handler.remoteClusterRequest(clusterId, req)
+               resp, err := h.handler.remoteClusterRequest(clusterID, req)
                h.handler.proxy.ForwardResponse(w, resp, err)
        }
 }
index 2da7ca5ccf98e14a6c2cb32d4602ecb81eb68101..5f96da56244325d86b3e9d4f252ec714f55f534c 100644 (file)
@@ -390,9 +390,8 @@ func (ta *oidcTokenAuthorizer) registerToken(ctx context.Context, tok string) er
                // cached negative result (value is expiry time)
                if time.Now().Before(exp) {
                        return nil
-               } else {
-                       ta.cache.Remove(tok)
                }
+               ta.cache.Remove(tok)
        } else {
                // cached positive result
                aca := cached.(arvados.APIClientAuthorization)
index 8cdba72c10d3c5902225456de9389bcc70b6dbca..da536107947187e3e88f1b59800a8d217666ca00 100644 (file)
@@ -162,7 +162,7 @@ func kill(uuid string, signal syscall.Signal, stdout, stderr io.Writer) error {
        return nil
 }
 
-// List UUIDs of active crunch-run processes.
+// ListProcesses lists UUIDs of active crunch-run processes.
 func ListProcesses(stdout, stderr io.Writer) int {
        // filepath.Walk does not follow symlinks, so we must walk
        // lockdir+"/." in case lockdir itself is a symlink.
index 901fda22897cd301d60539c372d77bc6815a799e..a81cf505294e0ce1bbc442da3b844d4cb5c5b073 100644 (file)
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-// package service provides a cmd.Handler that brings up a system service.
+// Package service provides a cmd.Handler that brings up a system service.
 package service
 
 import (
index 5e19d6b2cbef77a2a5b223b83e9321be35e78600..08fcfe3a3490739b0fcbf1bc8063fe4c7e60b94e 100644 (file)
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.summary     = "Arvados CLI tools"
   s.description = "Arvados command line tools, git commit #{git_hash}"
   s.authors     = ["Arvados Authors"]
-  s.email       = 'gem-dev@arvados.org'
+  s.email       = 'packaging@arvados.org'
   #s.bindir      = '.'
   s.licenses    = ['Apache-2.0']
   s.files       = ["bin/arv", "bin/arv-tag", "LICENSE-2.0.txt"]
index 50ebd25ff858dbef9cba4520f125d398d34a1e18..9a52ee70214d7f9474086818768c6cc8fbdafc17 100644 (file)
@@ -5,7 +5,7 @@
 fpm_depends+=(nodejs)
 
 case "$TARGET" in
-    debian9 | ubuntu1604)
+    ubuntu1604)
         fpm_depends+=(libcurl3-gnutls)
         ;;
     debian* | ubuntu*)
index d2dd6e8162e686355b6b4d5b076c54fad11c6637..517ca000bb3674817592855879a64370b4d6381c 100644 (file)
@@ -68,7 +68,7 @@ def stubs(func):
         stubs.keep_client = keep_client2
         stubs.docker_images = {
             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
-            "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
+            "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
         }
@@ -771,7 +771,7 @@ class TestSubmit(unittest.TestCase):
                                 ],
                                 'requirements': [
                                     {
-                                        'dockerPull': 'debian:8',
+                                        'dockerPull': 'debian:buster-slim',
                                         'class': 'DockerRequirement',
                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                                     }
@@ -1134,7 +1134,7 @@ class TestSubmit(unittest.TestCase):
                                 "hints": [
                                     {
                                         "class": "DockerRequirement",
-                                        "dockerPull": "debian:8",
+                                        "dockerPull": "debian:buster-slim",
                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                                     },
                                     {
index aadbd56351bd6f4b0dcfd29d44e70456b5ddba19..f8193d9f633644532a876c4e381a821b17571cd7 100644 (file)
@@ -11,7 +11,7 @@ class: CommandLineTool
 cwlVersion: v1.0
 requirements:
   - class: DockerRequirement
-    dockerPull: debian:8
+    dockerPull: debian:buster-slim
 inputs:
   - id: x
     type: File
index 0beb7ad78f7f740ef9d2512ab78251cd14be1ba0..c0c3c7a6b7469219e920d5082c557a06cf75361b 100644 (file)
@@ -11,7 +11,7 @@ class: CommandLineTool
 cwlVersion: v1.0
 requirements:
   - class: DockerRequirement
-    dockerPull: debian:8
+    dockerPull: debian:buster-slim
 inputs:
   - id: x
     type: File
index ce6f2c0c936fd91bb5583432cd8dfd1d2be34623..69054f569dc8d2203ba1c2a8dde08c9749b7764a 100644 (file)
@@ -7,7 +7,7 @@ cwlVersion: v1.0
 requirements:
   InlineJavascriptRequirement: {}
   DockerRequirement:
-    dockerPull: debian:stretch-slim
+    dockerPull: debian:buster-slim
 inputs:
   d: Directory
 outputs:
index 5739ddc7b40210a2eb84408f204fe3898f29fe60..116adcbf663097dbc6cda2bc0eadf2924231c0dd 100644 (file)
@@ -25,4 +25,4 @@ $graph:
     type: string
   outputs: []
   requirements:
-  - {class: DockerRequirement, dockerPull: 'debian:8'}
+  - {class: DockerRequirement, dockerPull: 'debian:buster-slim'}
index cb2e5ff56e10aee4b26162df2e07ddf4bca3f5f3..4715c10a5e27d92d2f59bba9cca220761d20a041 100644 (file)
@@ -25,7 +25,7 @@
             "requirements": [
                 {
                     "class": "DockerRequirement",
-                    "dockerPull": "debian:8",
+                    "dockerPull": "debian:buster-slim",
                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                 }
             ]
index 9a50fc8abdff5078a0f483098a2e264d553d4988..0b13e3a8192328b069c1057103cfe80f7e025f6a 100644 (file)
@@ -25,7 +25,7 @@
             "requirements": [
                 {
                     "class": "DockerRequirement",
-                    "dockerPull": "debian:8",
+                    "dockerPull": "debian:buster-slim",
                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
                 }
             ]
index 05d950d18c08be14ea72ea297585412136f8f198..5d2e231ec819887eac62ba0c23b1cbbfdc35dad4 100644 (file)
@@ -10,7 +10,7 @@ hints:
   "cwltool:Secrets":
     secrets: [pw]
   DockerRequirement:
-    dockerPull: debian:8
+    dockerPull: debian:buster-slim
 inputs:
   pw: string
 outputs:
index 83ba584b2084b39b3e507d203ab1bc4554ebda76..cd001703133eefe4afbc8c6162e7226f01249235 100644 (file)
@@ -7,7 +7,7 @@ $graph:
 - class: CommandLineTool
   requirements:
   - class: DockerRequirement
-    dockerPull: debian:8
+    dockerPull: debian:buster-slim
     'http://arvados.org/cwl#dockerCollectionPDH': 999999999999999999999999999999d4+99
   inputs:
   - id: '#submit_tool.cwl/x'
index f7719dbc4335048663e5195edaf791caa05dee8e..1e0068ffd48250375c60b8dc1331668a930fb809 100644 (file)
@@ -13,8 +13,8 @@
 # (This dockerfile file must be located in the arvados/sdk/ directory because
 #  of the docker build root.)
 
-FROM debian:9
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
 ENV DEBIAN_FRONTEND noninteractive
 
index 562c8c1e7d7c66528a2ce0874eca034c9eb7b328..5c6a7b52f43f97842bf207859fcfed4f4d057cce 100644 (file)
@@ -69,14 +69,14 @@ type Client struct {
        defaultRequestID string
 }
 
-// The default http.Client used by a Client with Insecure==true and
-// Client==nil.
+// InsecureHTTPClient is the default http.Client used by a Client with
+// Insecure==true and Client==nil.
 var InsecureHTTPClient = &http.Client{
        Transport: &http.Transport{
                TLSClientConfig: &tls.Config{
                        InsecureSkipVerify: true}}}
 
-// The default http.Client used by a Client otherwise.
+// DefaultSecureClient is the default http.Client used by a Client otherwise.
 var DefaultSecureClient = &http.Client{}
 
 // NewClientFromConfig creates a new Client that uses the endpoints in
index fdddfc537d8ee3b1dca86853232dc7017851969b..f7d1f35a3c322953c702437ca5caecd40687bddd 100644 (file)
@@ -17,7 +17,7 @@ type Link struct {
        Properties map[string]interface{} `json:"properties"`
 }
 
-// UserList is an arvados#userList resource.
+// LinkList is an arvados#linkList resource.
 type LinkList struct {
        Items          []Link `json:"items"`
        ItemsAvailable int    `json:"items_available"`
index e65baa3b3f48460329a49f95782b664b0bc150e0..d600322e6a8463a7a7a66577683aad6c697fa031 100644 (file)
@@ -2,7 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-/* Provides low-level Get/Put primitives for accessing Arvados Keep blocks. */
+// Package keepclient provides low-level Get/Put primitives for accessing
+// Arvados Keep blocks.
 package keepclient
 
 import (
@@ -25,7 +26,7 @@ import (
        "git.arvados.org/arvados.git/sdk/go/httpserver"
 )
 
-// A Keep "block" is 64MB.
+// BLOCKSIZE defines the length of a Keep "block", which is 64MB.
 const BLOCKSIZE = 64 * 1024 * 1024
 
 var (
@@ -82,8 +83,8 @@ var ErrNoSuchKeepServer = errors.New("No keep server matching the given UUID is
 // ErrIncompleteIndex is returned when the Index response does not end with a new empty line
 var ErrIncompleteIndex = errors.New("Got incomplete index")
 
-const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
-const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
+const XKeepDesiredReplicas = "X-Keep-Desired-Replicas"
+const XKeepReplicasStored = "X-Keep-Replicas-Stored"
 
 type HTTPClient interface {
        Do(*http.Request) (*http.Response, error)
@@ -139,7 +140,7 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
        }
 }
 
-// Put a block given the block hash, a reader, and the number of bytes
+// PutHR puts a block given the block hash, a reader, and the number of bytes
 // to read from the reader (which must be between 0 and BLOCKSIZE).
 //
 // Returns the locator for the written block, the number of replicas
@@ -216,7 +217,7 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade
 
        var errs []string
 
-       tries_remaining := 1 + kc.Retries
+       triesRemaining := 1 + kc.Retries
 
        serversToTry := kc.getSortedRoots(locator)
 
@@ -225,8 +226,8 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade
 
        var retryList []string
 
-       for tries_remaining > 0 {
-               tries_remaining -= 1
+       for triesRemaining > 0 {
+               triesRemaining -= 1
                retryList = nil
 
                for _, host := range serversToTry {
index f4e99bdbe3ef8c699d1bd7aadf97a861f5950b1d..6a31c98bc6596a5f946fc73ec0a8d22a66238b80 100644 (file)
@@ -79,7 +79,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
        req.Header.Add("X-Request-Id", reqid)
        req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
        req.Header.Add("Content-Type", "application/octet-stream")
-       req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
+       req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(this.Want_replicas))
        if len(this.StorageClasses) > 0 {
                req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
        }
@@ -92,7 +92,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
        }
 
        rep := 1
-       if xr := resp.Header.Get(X_Keep_Replicas_Stored); xr != "" {
+       if xr := resp.Header.Get(XKeepReplicasStored); xr != "" {
                fmt.Sscanf(xr, "%d", &rep)
        }
 
index a03d6afe6abbaad5932f88de50e4f486f7deefdd..570e398a2895ffb61ff021e0f7a618e3c21051d6 100644 (file)
@@ -39,11 +39,11 @@ Installing on Debian systems
 
 1. Add this Arvados repository to your sources list::
 
-     deb http://apt.arvados.org/ stretch main
+     deb http://apt.arvados.org/ buster main
 
 2. Update your package list.
 
-3. Install the ``python-arvados-python-client`` package.
+3. Install the ``python3-arvados-python-client`` package.
 
 Configuration
 -------------
index 71db40c7fd6bc7f5eef8eb6f757e80466b6d3b9f..7cc2fd931c8cc3bd7c0446953e0eb49b23f09f85 100644 (file)
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.summary     = "Arvados client library"
   s.description = "Arvados client library, git commit #{git_hash}"
   s.authors     = ["Arvados Authors"]
-  s.email       = 'gem-dev@curoverse.com'
+  s.email       = 'packaging@arvados.org'
   s.licenses    = ['Apache-2.0']
   s.files       = ["lib/arvados.rb", "lib/arvados/google_api_client.rb",
                    "lib/arvados/collection.rb", "lib/arvados/keep.rb",
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.service b/services/crunch-dispatch-local/crunch-dispatch-local.service
new file mode 100644 (file)
index 0000000..692d81e
--- /dev/null
@@ -0,0 +1,29 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+[Unit]
+Description=Arvados Crunch Dispatcher for LOCAL service
+Documentation=https://doc.arvados.org/
+After=network.target
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
+StartLimitInterval=0
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+EnvironmentFile=-/etc/arvados/crunch-dispatch-local-credentials
+ExecStart=/usr/bin/crunch-dispatch-local -poll-interval=1 -crunch-run-command=/usr/bin/crunch-run
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
+Restart=always
+RestartSec=1
+LimitNOFILE=1000000
+
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target
diff --git a/services/crunch-dispatch-local/fpm-info.sh b/services/crunch-dispatch-local/fpm-info.sh
new file mode 100644 (file)
index 0000000..6956c4c
--- /dev/null
@@ -0,0 +1,5 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+fpm_depends+=(crunch-run)
index d678fdfd7a89206c75af6a5080f434fd9c7a72f1..ccb7a467af45906f60710c94e0dfe98bf84bf034 100644 (file)
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 case "$TARGET" in
-    debian9 | ubuntu1604)
+    ubuntu1604)
         fpm_depends+=()
         ;;
     debian* | ubuntu*)
index d3db94e5db6c2ad62f59c347c441ad821b0ef1bb..d528dbaf79894807ba0120d58dd1fbd65c60cbd4 100644 (file)
@@ -34,9 +34,8 @@ func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) {
                                URL:           "http://" + s.testServer.Addr,
                                SigningRegion: "custom-signing-region",
                        }, nil
-               } else {
-                       return endpoints.NewDefaultResolver().ResolveEndpoint(service, region)
                }
+               return endpoints.NewDefaultResolver().ResolveEndpoint(service, region)
        })
        client := s3.New(cfg)
        client.ForcePathStyle = true
index 4e5014ef822d7551fc0e3db960b5c6d42f876f63..538a0612275ec029e448b810f45bcdd08fee74bb 100644 (file)
@@ -173,37 +173,42 @@ type APITokenCache struct {
        expireTime int64
 }
 
-// Cache the token and set an expire time.  If we already have an expire time
-// on the token, it is not updated.
-func (this *APITokenCache) RememberToken(token string) {
-       this.lock.Lock()
-       defer this.lock.Unlock()
+// RememberToken caches the token and set an expire time.  If we already have
+// an expire time on the token, it is not updated.
+func (cache *APITokenCache) RememberToken(token string) {
+       cache.lock.Lock()
+       defer cache.lock.Unlock()
 
        now := time.Now().Unix()
-       if this.tokens[token] == 0 {
-               this.tokens[token] = now + this.expireTime
+       if cache.tokens[token] == 0 {
+               cache.tokens[token] = now + cache.expireTime
        }
 }
 
-// Check if the cached token is known and still believed to be valid.
-func (this *APITokenCache) RecallToken(token string) bool {
-       this.lock.Lock()
-       defer this.lock.Unlock()
+// RecallToken checks if the cached token is known and still believed to be
+// valid.
+func (cache *APITokenCache) RecallToken(token string) bool {
+       cache.lock.Lock()
+       defer cache.lock.Unlock()
 
        now := time.Now().Unix()
-       if this.tokens[token] == 0 {
+       if cache.tokens[token] == 0 {
                // Unknown token
                return false
-       } else if now < this.tokens[token] {
+       } else if now < cache.tokens[token] {
                // Token is known and still valid
                return true
        } else {
                // Token is expired
-               this.tokens[token] = 0
+               cache.tokens[token] = 0
                return false
        }
 }
 
+// GetRemoteAddress returns a string with the remote address for the request.
+// If the X-Forwarded-For header is set and has a non-zero length, it returns a
+// string made from a comma separated list of all the remote addresses,
+// starting with the one(s) from the X-Forwarded-For header.
 func GetRemoteAddress(req *http.Request) string {
        if xff := req.Header.Get("X-Forwarded-For"); xff != "" {
                return xff + "," + req.RemoteAddr
@@ -507,7 +512,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) {
        // Check if the client specified the number of replicas
        if req.Header.Get("X-Keep-Desired-Replicas") != "" {
                var r int
-               _, err := fmt.Sscanf(req.Header.Get(keepclient.X_Keep_Desired_Replicas), "%d", &r)
+               _, err := fmt.Sscanf(req.Header.Get(keepclient.XKeepDesiredReplicas), "%d", &r)
                if err == nil {
                        kc.Want_replicas = r
                }
@@ -527,7 +532,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) {
        }
 
        // Tell the client how many successful PUTs we accomplished
-       resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", wroteReplicas))
+       resp.Header().Set(keepclient.XKeepReplicasStored, fmt.Sprintf("%d", wroteReplicas))
 
        switch err.(type) {
        case nil:
index 8e338d2f2b697f47b45f1d3003d7c5c6f4607873..911548bbf932b12bdf3acbaaf550dc01ce2786a5 100644 (file)
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.summary     = "Set up local login accounts for Arvados users"
   s.description = "Creates and updates local login accounts for Arvados users. Built from git commit #{git_hash}"
   s.authors     = ["Arvados Authors"]
-  s.email       = 'gem-dev@curoverse.com'
+  s.email       = 'packaging@arvados.org'
   s.licenses    = ['AGPL-3.0']
   s.files       = ["bin/arvados-login-sync", "agpl-3.0.txt"]
   s.executables << "arvados-login-sync"