X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/861cff5ffc2eb2739573b8991fca1cfdc388377f..9f0854e394baf712f1fbcf3ebd21af215276c0fb:/build/run-build-packages.sh

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 697092966b..300333a0b7 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -15,8 +15,10 @@ Options:
     Build api server and workbench packages with vendor/bundle included
 --debug
     Output debug information (default: false)
---target
+--target <target>
     Distribution to build packages for (default: debian7)
+--only-build <package>
+    Build only a specific package (or $ONLY_BUILD from environment)
 --command
     Build command to execute (defaults to the run command defined in the
     Docker image)
@@ -31,7 +33,7 @@ TARGET=debian7
 COMMAND=
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,build-bundle-packages,debug,target: \
+    help,build-bundle-packages,debug,target:,only-build: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -48,6 +50,9 @@ while [ $# -gt 0 ]; do
         --target)
             TARGET="$2"; shift
             ;;
+        --only-build)
+            ONLY_BUILD="$2"; shift
+            ;;
         --debug)
             DEBUG=1
             ;;
@@ -82,70 +87,101 @@ declare -a PYTHON_BACKPORTS PYTHON3_BACKPORTS
 PYTHON2_VERSION=2.7
 PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
 
+## These defaults are suitable for any Debian-based distribution.
+# You can customize them as needed in distro sections below.
+PYTHON2_PACKAGE=python$PYTHON2_VERSION
+PYTHON2_PKG_PREFIX=python
+PYTHON2_PREFIX=/usr
+PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/dist-packages
+
+PYTHON3_PACKAGE=python$PYTHON3_VERSION
+PYTHON3_PKG_PREFIX=python3
+PYTHON3_PREFIX=/usr
+PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
+## End Debian Python defaults.
+
 case "$TARGET" in
     debian7)
         FORMAT=deb
-        PYTHON2_PACKAGE=python$PYTHON2_VERSION
-        PYTHON2_PKG_PREFIX=python
-        PYTHON3_PACKAGE=python$PYTHON3_VERSION
-        PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
             oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six  \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5' contextlib2)
-        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+            'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
+            shellescape mistune typing avro ruamel.ordereddict
+            cachecontrol requests)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     debian8)
         FORMAT=deb
-        PYTHON2_PACKAGE=python$PYTHON2_VERSION
-        PYTHON2_PKG_PREFIX=python
-        PYTHON3_PACKAGE=python$PYTHON3_VERSION
-        PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
             oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six  \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            'pycurl<7.21.5')
-        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+            'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
+            shellescape mistune typing avro ruamel.ordereddict
+            cachecontrol)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1204)
         FORMAT=deb
-        PYTHON2_PACKAGE=python$PYTHON2_VERSION
-        PYTHON2_PKG_PREFIX=python
-        PYTHON3_PACKAGE=python$PYTHON3_VERSION
-        PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
             oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six  \
             ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
-            contextlib2 \
-            'pycurl<7.21.5')
-        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+            contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
+            shellescape mistune typing avro isodate ruamel.ordereddict
+            cachecontrol requests)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         ;;
     ubuntu1404)
         FORMAT=deb
-        PYTHON2_PACKAGE=python$PYTHON2_VERSION
-        PYTHON2_PKG_PREFIX=python
-        PYTHON3_PACKAGE=python$PYTHON3_VERSION
-        PYTHON3_PKG_PREFIX=python3
-        PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
+        PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \
             google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
-            rsa 'pycurl<7.21.5' backports.ssl_match_hostname)
-        PYTHON3_BACKPORTS=(docker-py requests websocket-client)
+            rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
+            shellescape mistune typing avro ruamel.ordereddict
+            cachecontrol)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
         ;;
     centos6)
         FORMAT=rpm
         PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
         PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+        PYTHON2_PREFIX=/opt/rh/python27/root/usr
+        PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
-        PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client==1.4.2 \
+        PYTHON3_PREFIX=/opt/rh/python33/root/usr
+        PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
+        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
             oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
-            rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+            rsa uritemplate httplib2 ws4py pykka six  \
             ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
-            python-daemon lockfile llfuse==0.41.1 'pbr<1.0')
-        PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
+            python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
+            'rdflib>=4.2.0' shellescape mistune typing avro requests \
+            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
+            ruamel.ordereddict cachecontrol)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
+        export PYCURL_SSL_LIBRARY=nss
+        ;;
+    centos7)
+        FORMAT=rpm
+        PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
+        PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
+        PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages
+        PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
+        PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
+        PYTHON3_PREFIX=/opt/rh/python33/root/usr
+        PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
+        PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
+            oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
+            rsa uritemplate httplib2 ws4py pykka  \
+            ciso8601 pycrypto 'pycurl<7.21.5' \
+            python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml \
+            'rdflib>=4.2.0' shellescape mistune typing avro \
+            isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
+            ruamel.ordereddict cachecontrol)
+        PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
         export PYCURL_SSL_LIBRARY=nss
         ;;
     *)
@@ -200,6 +236,7 @@ fi
 chmod o+r "$WORKSPACE" -R
 
 # More cleanup - make sure all executables that we'll package are 755
+cd "$WORKSPACE"
 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 
 # Now fix our umask to something better suited to building and publishing
@@ -210,11 +247,13 @@ debug_echo "umask is" `umask`
 
 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
   mkdir -p $WORKSPACE/packages/$TARGET
+  chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
 fi
 
 # Perl packages
 debug_echo -e "\nPerl packages\n"
 
+if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
 cd "$WORKSPACE/sdk/perl"
 
 if [[ -e Makefile ]]; then
@@ -230,6 +269,7 @@ perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
     "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
     "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
     mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+fi
 
 # Ruby gems
 debug_echo -e "\nRuby gems\n"
@@ -351,7 +391,7 @@ elif [[ $TARGET =~ centos6 ]]; then
             rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i
             perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec
         )
-        # build rpms from source 
+        # build rpms from source
         rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec
         rm -f fuse-2.9.2-6.el7.src.rpm
         # move built RPMs to LIBFUSE_DIR
@@ -372,20 +412,34 @@ fi
 # Go binaries
 cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
-package_go_binary services/keepstore keepstore \
-    "Keep storage daemon, accessible to clients on the LAN"
-package_go_binary services/keepproxy keepproxy \
-    "Make a Keep cluster accessible to clients that are not on the LAN"
-package_go_binary services/keep-web keep-web \
-    "Static web hosting service for user data stored in Arvados Keep"
-package_go_binary services/datamanager arvados-data-manager \
-    "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary sdk/go/crunchrunner crunchrunner \
+    "Crunchrunner executes a command inside a container and uploads the output"
 package_go_binary services/arv-git-httpd arvados-git-httpd \
     "Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
+    "Dispatch Crunch containers on the local system"
+package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
+    "Dispatch Crunch containers to a SLURM cluster"
+package_go_binary services/crunch-run crunch-run \
+    "Supervise a single Crunch container"
 package_go_binary services/crunchstat crunchstat \
     "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary services/datamanager arvados-data-manager \
+    "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary services/keep-balance keep-balance \
+    "Rebalance and garbage-collect data blocks stored in Arvados Keep"
+package_go_binary services/keepproxy keepproxy \
+    "Make a Keep cluster accessible to clients that are not on the LAN"
+package_go_binary services/keepstore keepstore \
+    "Keep storage daemon, accessible to clients on the LAN"
+package_go_binary services/keep-web keep-web \
+    "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary tools/keep-block-check keep-block-check \
+    "Verify that all data from one set of Keep servers to another was copied"
 package_go_binary tools/keep-rsync keep-rsync \
     "Copy all data from one set of Keep servers to another"
+package_go_binary tools/keep-exercise keep-exercise \
+    "Performance testing tool for Arvados Keep"
 
 # The Python SDK
 # Please resist the temptation to add --no-python-fix-name to the fpm call here
@@ -401,7 +455,42 @@ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'C
 # cwl-runner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner"
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
+
+fpm_build lockfile "" "" python 0.12.2 --epoch 1
+
+# schema_salad. This is a python dependency of arvados-cwl-runner,
+# but we can't use the usual PYTHONPACKAGES way to build this package due to the
+# intricacies of how version numbers get generated in setup.py: we need a specific version,
+# e.g. 1.7.20160316203940. If we don't explicitly list that version with the -v
+# argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get
+# a package with version 1.7. That's because our gittagger hack is not being
+# picked up by self.distribution.get_version(), which is called from
+# https://github.com/jordansissel/fpm/blob/master/lib/fpm/package/pyfpm/get_metadata.py
+# by means of this command:
+#
+# python2.7 setup.py --command-packages=pyfpm get_metadata --output=metadata.json
+#
+# So we build this thing separately.
+#
+# Ward, 2016-03-17
+saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
+fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+
+# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
+# Ward, 2016-05-26
+fpm_build ruamel.yaml "" "" python 0.12.4 --python-setup-py-arguments "--single-version-externally-managed"
+
+# Dependency of cwltool.  Fpm doesn't produce a package with the correct version
+# number unless we build it explicitly
+fpm_build cwltest "" "" python 1.0.20160907111242
+
+# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
+cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
+fpm_build cwltool "" "" python $cwltoolversion
+
+# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
+fpm_build rdflib-jsonld "" "" python 0.3.0
 
 # The PAM module
 if [[ $TARGET =~ debian|ubuntu ]]; then
@@ -425,7 +514,12 @@ fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.'
 # The Docker image cleaner
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
-fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner"
+fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3
+
+# The Arvados crunchstat-summary tool
+cd $WORKSPACE/packages/$TARGET
+rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
+fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
 
 # Forked libcloud
 LIBCLOUD_DIR=$(mktemp -d)
@@ -454,6 +548,11 @@ esac
 
 for deppkg in "${PYTHON_BACKPORTS[@]}"; do
     outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
+
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then
+        continue
+    fi
+
     case "$deppkg" in
         httplib2|google-api-python-client)
             # Work around 0640 permissions on some package files.
@@ -462,14 +561,15 @@ for deppkg in "${PYTHON_BACKPORTS[@]}"; do
                 set -e
                 cd "$pyfpm_workdir"
                 pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
-                tar -xf "$deppkg"-*.tar*
+                # Sometimes pip gives us a tarball, sometimes a zip file...
+                DOWNLOADED=`ls $deppkg-*`
+                [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED
+                [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED
                 cd "$deppkg"-*/
                 "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
                 chmod -R go+rX .
                 set +e
-                # --iteration 2 provides an upgrade for previously built
-                # buggy packages.
-                fpm_build . "$outname" "" python "" --iteration 2
+                fpm_build . "$outname" "" python "" --iteration 3
                 # The upload step uses the package timestamp to determine
                 # whether it's new.  --no-clobber plays nice with that.
                 mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
@@ -502,6 +602,7 @@ handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
     --license="GNU Affero General Public License, version 3.0"
 
 # Build the workbench server package
+if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then
 (
     set -e
     cd "$WORKSPACE/apps/workbench"
@@ -526,6 +627,7 @@ handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
     # Remove generated configuration files so they don't go in the package.
     rm config/application.yml config/environments/production.rb
 )
+fi
 
 if [[ "$?" != "0" ]]; then
   echo "ERROR: Asset precompilation failed"