+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- ubuntu1204)
- fpm_depends+=('libfuse2 = 2.9.2-5')
- ;;
-esac
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- ubuntu1204)
- fpm_depends+=('libfuse2 = 2.9.2-5')
- ;;
-esac
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case $TARGET in
- centos7)
- # fpm incorrectly transforms the dependency name in this case.
- fpm_depends+=(python-backports-ssl_match_hostname)
- fpm_args+=(--python-disable-dependency backports.ssl-match-hostname)
- ;;
-esac
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- centos*)
- fpm_depends+=(glibc)
- ;;
- debian* | ubuntu*)
- fpm_depends+=(libc6)
- ;;
-esac
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- centos*)
- build_depends+=('fuse-devel')
- fpm_depends+=(glibc fuse-libs)
- ;;
- ubuntu1204)
- build_depends+=(libfuse2 libfuse-dev)
- fpm_depends+=(libc6 python-contextlib2 'libfuse2 = 2.9.2-5' 'fuse = 2.9.2-5')
- ;;
- debian* | ubuntu*)
- build_depends+=('libfuse-dev')
- fpm_depends+=(libc6 'libfuse2 > 2.9.0' 'fuse > 2.9.0')
- ;;
-esac
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- centos*)
- fpm_depends+=(glibc)
- ;;
- debian8)
- fpm_depends+=(libc6 libgmp10)
- ;;
- debian* | ubuntu*)
- fpm_depends+=(libc6)
- ;;
-esac
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-case "$TARGET" in
- debian8)
- fpm_depends+=(
- libc6
- libcomerr2
- libcurl3-gnutls
- libffi6
- libgcrypt20
- libgmp10
- libgnutls-deb0-28
- libgpg-error0
- libgssapi-krb5-2
- libhogweed2
- libidn11
- libk5crypto3
- libkeyutils1
- libkrb5-3
- libkrb5support0
- libldap-2.4-2
- libnettle4
- libp11-kit0
- librtmp1
- libsasl2-2
- libssh2-1
- libtasn1-6
- zlib1g
- ) ;;
- ubuntu1204)
- fpm_depends+=(
- libasn1-8-heimdal
- libc6
- libcomerr2
- libcurl3-gnutls
- libgcrypt11
- libgnutls26
- libgpg-error0
- libgssapi-krb5-2
- libgssapi3-heimdal
- libhcrypto4-heimdal
- libheimbase1-heimdal
- libheimntlm0-heimdal
- libhx509-5-heimdal
- libidn11
- libk5crypto3
- libkeyutils1
- libkrb5-26-heimdal
- libkrb5-3
- libkrb5support0
- libldap-2.4-2
- libp11-kit0
- libroken18-heimdal
- librtmp0
- libsasl2-2
- libsqlite3-0
- libtasn1-3
- libwind0-heimdal
- zlib1g
- ) ;;
- ubuntu1404)
- fpm_depends+=(
- libasn1-8-heimdal
- libc6
- libcomerr2
- libcurl3-gnutls
- libffi6
- libgcrypt11
- libgnutls26
- libgpg-error0
- libgssapi-krb5-2
- libgssapi3-heimdal
- libhcrypto4-heimdal
- libheimbase1-heimdal
- libheimntlm0-heimdal
- libhx509-5-heimdal
- libidn11
- libk5crypto3
- libkeyutils1
- libkrb5-26-heimdal
- libkrb5-3
- libkrb5support0
- libldap-2.4-2
- libp11-kit0
- libroken18-heimdal
- librtmp0
- libsasl2-2
- libsqlite3-0
- libtasn1-6
- libwind0-heimdal
- zlib1g
- ) ;;
-esac
Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job)
want=${wantGPL}
;;
- crunch_scripts/* | backports/* | docker/* | sdk/*)
+ crunch_scripts/* | docker/* | sdk/*)
want=${wantApache}
;;
doc/*)
EOF
-EXITCODE=0
+# Begin of user configuration
+
+# set to --no-cache-dir to disable pip caching
+CACHE_FLAG=
+
+MAINTAINER="Ward Vandewege <wvandewege@veritasgenetics.com>"
+VENDOR="Veritas Genetics, Inc."
+
+# End of user configuration
+
DEBUG=${ARVADOS_DEBUG:-0}
+EXITCODE=0
TARGET=debian8
COMMAND=
perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
fpm_build install/lib/=/usr/share libarvados-perl \
- "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+ dir "$(version_from_git)" install/man/=/usr/share/man \
"$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
fi
cd "$SRC_BUILD_DIR"
PKG_VERSION=$(version_from_git)
cd $WORKSPACE/packages/$TARGET
- fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+ fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
rm -rf "$SRC_BUILD_DIR"
-
fi
)
fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl"
# The PAM module
-if [[ $TARGET =~ debian|ubuntu ]]; then
- cd $WORKSPACE/packages/$TARGET
- rm -rf "$WORKSPACE/sdk/pam/build"
- libpam_arvados_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)
- test_package_presence libpam-arvados "$libpam_arvados_version" python
- if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$libpam_arvados_version" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python
- fi
-fi
+fpm_build_virtualenv "libpam-arvados" "sdk/pam"
# The FUSE driver
fpm_build_virtualenv "arvados-fuse" "services/fuse"
# The Arvados crunchstat-summary tool
fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary"
-# Forked libcloud
-if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
-then
- LIBCLOUD_DIR=$(mktemp -d)
- (
- cd $LIBCLOUD_DIR
- git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git .
- git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN
- # libcloud is absurdly noisy without -q, so force -q here
- OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG
- DASHQ_UNLESS_DEBUG=-q
- handle_python_package
- DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
- )
-
- # libcloud >= 2.3.0 now requires python-requests 2.4.3 or higher, otherwise
- # it throws
- # ImportError: No module named packages.urllib3.poolmanager
- # when loaded. We only see this problem on ubuntu1404, because that is our
- # only supported distribution that ships with a python-requests older than
- # 2.4.3.
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2 --depends 'python-requests >= 2.4.3'
- rm -rf $LIBCLOUD_DIR
-fi
-
# Build the API server package
test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
if [[ "$?" == "0" ]]; then
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" 'Curoverse, Inc.' dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
default_iteration() {
return 1
fi
local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" dir "$version")
local license_arg="$license_path=$railsdir/$(basename "$license_path")"
local -a switches=(--after-install "$scripts_dir/postinst"
--before-remove "$scripts_dir/prerm"
COMMAND_ARR+=("${fpm_args[@]}")
- # Make sure to install all our package binaries in /usr/local/bin.
+ # Make sure to install all our package binaries in /usr/bin.
# We have to walk $WORKSPACE/$PKG_DIR/bin rather than
# $WORKSPACE/build/usr/share/$python/dist/$PYTHON_PKG/bin/ to get the list
# because the latter also includes all the python binaries for the virtualenv.
# because those are the ones we rewrote the shebang line of, above.
if [[ -e "$WORKSPACE/$PKG_DIR/bin" ]]; then
for binary in `ls $WORKSPACE/$PKG_DIR/bin`; do
- COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/$binary=/usr/local/bin/")
+ COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/$binary=/usr/bin/")
done
fi
# The name of the package to build.
PACKAGE_NAME=$1
shift
- # Optional: the vendor of the package. Should be "Curoverse, Inc." for
- # packages of our own software. Passed to fpm --vendor.
- VENDOR=$1
- shift
- # The type of source package. Passed to fpm -s. Default "python".
- PACKAGE_TYPE=${1:-python}
+ # The type of source package. Passed to fpm -s. Default "dir".
+ PACKAGE_TYPE=${1:-dir}
shift
# Optional: the package version number. Passed to fpm -v.
VERSION=$1
fi
local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
- local python=""
- case "$PACKAGE_TYPE" in
- python)
- # All Arvados Python2 packages depend on Python 2.7.
- # Make sure we build with that for consistency.
- python=python2.7
- set -- "$@" --python-bin python2.7 \
- "${PYTHON_FPM_INSTALLER[@]}" \
- --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
- --prefix "$PYTHON2_PREFIX" \
- --python-install-lib "$PYTHON2_INSTALL_LIB" \
- --python-install-data . \
- --exclude "${PYTHON2_INSTALL_LIB#/}/tests" \
- --depends "$PYTHON2_PACKAGE"
- ;;
- python3)
- # fpm does not actually support a python3 package type. Instead
- # we recognize it as a convenience shortcut to add several
- # necessary arguments to fpm's command line later, after we're
- # done handling positional arguments.
- PACKAGE_TYPE=python
- python=python3
- set -- "$@" --python-bin python3 \
- "${PYTHON3_FPM_INSTALLER[@]}" \
- --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
- --prefix "$PYTHON3_PREFIX" \
- --python-install-lib "$PYTHON3_INSTALL_LIB" \
- --python-install-data . \
- --exclude "${PYTHON3_INSTALL_LIB#/}/tests" \
- --depends "$PYTHON3_PACKAGE"
- ;;
- esac
-
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
+ declare -a COMMAND_ARR=("fpm" "-s" "$PACKAGE_TYPE" "-t" "$FORMAT")
if [ python = "$PACKAGE_TYPE" ] && [ deb = "$FORMAT" ]; then
# Dependencies are built from setup.py. Since setup.py will never
# refer to Debian package iterations, it doesn't make sense to
COMMAND_ARR+=('-n' "$PACKAGE_NAME")
fi
+ if [[ "$MAINTAINER" != "" ]]; then
+ COMMAND_ARR+=('--maintainer' "$MAINTAINER")
+ fi
+
if [[ "$VENDOR" != "" ]]; then
COMMAND_ARR+=('--vendor' "$VENDOR")
fi
COMMAND_ARR+=(--iteration "$default_iteration_value")
fi
- if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
- then
- COMMAND_ARR+=(
- --after-install "${WORKSPACE}/build/go-python-package-scripts/postinst"
- --before-remove "${WORKSPACE}/build/go-python-package-scripts/prerm"
- )
- fi
-
# Append --depends X and other arguments specified by fpm-info.sh in
# the package source dir. These are added last so they can override
# the arguments added by this script.
declare -a fpm_exclude=()
declare -a fpm_dirs=(
# source dir part of 'dir' package ("/source=/dest" => "/source"):
- "${PACKAGE%%=/*}"
- # backports ("llfuse>=1.0" => "backports/python-llfuse")
- "${WORKSPACE}/backports/${PACKAGE_TYPE}-${PACKAGE%%[<=>]*}")
- if [[ -n "$PACKAGE_NAME" ]]; then
- fpm_dirs+=("${WORKSPACE}/backports/${PACKAGE_NAME}")
- fi
+ "${PACKAGE%%=/*}")
for pkgdir in "${fpm_dirs[@]}"; do
fpminfo="$pkgdir/fpm-info.sh"
if [[ -e "$fpminfo" ]]; then
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
+h3. current master branch
+
+h4. Python packaging change
+
+As part of story "#9945":https://dev.arvados.org/issues/9945, the distribution packaging (deb/rpm) of our Python packages has changed. These packages now include a built-in virtualenv, which means fewer dependencies on system packages. We have also stopped packaging and publishing backports for all the Python dependencies of our packages, as they are no longer needed.
+
+One practical consequence of this change is that the use of the Arvados Python SDK (aka "import arvados"), if the SDK was installed from distribution packages, now requires the loading of the virtualenv environment from our packages. The "Install documentation for the Arvados Python SDK":https:doc.arvados.org/sdk/python/sdk-python.html reflects this change. This does not affect the use of our command line tools.
+
h3. v1.2.1 (2018-11-26)
There are no special upgrade notes for this release.
The Python SDK provides access from Python to the Arvados API and Keep. It also includes a number of command line tools for using and administering Arvados and Keep, and some conveniences for use in Crunch scripts; see "Crunch utility libraries":crunch-utility-libraries.html for details.
-h3. Installation
+h2. Installation
If you are logged in to an Arvados VM, the Python SDK should be installed.
The Python SDK requires Python 2.7.
{% include 'notebox_end' %}
-h4. Option 1: Install from distribution packages
+h3. Option 1: Install with pip
-First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
+Run @pip-2.7 install arvados-python-client@ in an appropriate installation environment, such as a virtualenv.
+
+If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip-2.7 install --pre arvados-python-client@.
+
+h3. Option 2: Install from source
-{% assign rh_version = "6" %}
-{% include 'note_python_sc' %}
+Install the @python-setuptools@ package from your distribution. Then run the following:
+
+<notextile>
+<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
+~$ <span class="userinput">cd arvados/sdk/python</span>
+~/arvados/sdk/python$ <span class="userinput">python2.7 setup.py install</span>
+</code></pre>
+</notextile>
+
+You may optionally run the final installation command in a virtualenv, or with the @--user@ option.
+
+h3. Option 3: Install from distribution packages
+
+First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
On Red Hat-based systems:
</code></pre>
</notextile>
-h4. Option 2: Install with pip
+h3. Test installation
-Run @pip-2.7 install arvados-python-client@ in an appropriate installation environment, such as a virtualenv.
+If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors.
-If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip-2.7 install --pre arvados-python-client@.
-
-h4. Option 3: Install from source
-
-Install the @python-setuptools@ package from your distribution. Then run the following:
+If you installed with pip or from source (option 1 or 2 above):
<notextile>
-<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
-~$ <span class="userinput">cd arvados/sdk/python</span>
-~/arvados/sdk/python$ <span class="userinput">python2.7 setup.py install</span>
-</code></pre>
+<pre>~$ <code class="userinput">python</code>
+Python 2.7.4 (default, Sep 26 2013, 03:20:26)
+[GCC 4.7.3] on linux2
+Type "help", "copyright", "credits" or "license" for more information.
+>>> <code class="userinput">import arvados</code>
+>>> <code class="userinput">arvados.api('v1')</code>
+<apiclient.discovery.Resource object at 0x233bb50>
+</pre>
</notextile>
-You may optionally run the final installation command in a virtualenv, or with the @--user@ option.
-
-h4. Test installation
-
-If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors:
+If you installed from distribution packages (option 3): the packages include a virtualenv, which means the correct python environment needs to be loaded to import the Arvados SDK:
<notextile>
-<pre>~$ <code class="userinput">python2.7</code>
+<pre>~$ <code class="userinput">/usr/share/python2.7/dist/python-arvados-python-client/bin/python</code>
Python 2.7.4 (default, Sep 26 2013, 03:20:26)
[GCC 4.7.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
case "$TARGET" in
debian* | ubuntu*)
- fpm_depends+=('libpam-python')
+ fpm_depends+=('libpam-python' 'libcurl3-gnutls')
;;
*)
echo >&2 "ERROR: $PACKAGE: pam_python.so dependency unavailable in $TARGET."
projects / arvados.git / commitdiff