X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a95f899d7ac84f29b3d019aa410d265bb40833e5..9f35a3cf10e5d261d83556d5b33f242966101f02:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index e7934a6484..65c928239f 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -100,30 +100,33 @@ case "$TARGET" in FORMAT=deb PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + rsa uritemplate httplib2 ws4py pykka six \ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ 'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict) + shellescape mistune typing avro ruamel.ordereddict + cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) ;; debian8) FORMAT=deb PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + rsa uritemplate httplib2 ws4py pykka six \ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict) + shellescape mistune typing avro ruamel.ordereddict + cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) ;; ubuntu1204) FORMAT=deb PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + rsa uritemplate httplib2 ws4py pykka six \ ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro isodate ruamel.ordereddict) + shellescape mistune typing avro isodate ruamel.ordereddict + cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) ;; ubuntu1404) @@ -131,7 +134,8 @@ case "$TARGET" in PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \ google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \ rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict) + shellescape mistune typing avro ruamel.ordereddict + cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client) ;; centos6) @@ -146,12 +150,12 @@ case "$TARGET" in PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + rsa uritemplate httplib2 ws4py pykka six \ ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \ - python-daemon lockfile llfuse==0.41.1 'pbr<1.0' pyyaml \ + python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \ 'rdflib>=4.2.0' shellescape mistune typing avro requests \ - isodate pyparsing sparqlwrapper html5lib keepalive \ - ruamel.ordereddict) + isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \ + ruamel.ordereddict cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) export PYCURL_SSL_LIBRARY=nss ;; @@ -166,12 +170,12 @@ case "$TARGET" in PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka pyexecjs jsonschema \ + rsa uritemplate httplib2 ws4py pykka \ ciso8601 pycrypto 'pycurl<7.21.5' \ python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \ 'rdflib>=4.2.0' shellescape mistune typing avro \ - isodate pyparsing sparqlwrapper html5lib keepalive \ - ruamel.ordereddict) + isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \ + ruamel.ordereddict cachecontrol) PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client) export PYCURL_SSL_LIBRARY=nss ;; @@ -238,6 +242,7 @@ debug_echo "umask is" `umask` if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then mkdir -p $WORKSPACE/packages/$TARGET + chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET" fi # Perl packages @@ -400,26 +405,32 @@ fi # Go binaries cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) -package_go_binary services/keepstore keepstore \ - "Keep storage daemon, accessible to clients on the LAN" -package_go_binary services/keepproxy keepproxy \ - "Make a Keep cluster accessible to clients that are not on the LAN" -package_go_binary services/keep-balance keep-balance \ - "Rebalance and garbage-collect data blocks stored in Arvados Keep" -package_go_binary services/keep-web keep-web \ - "Static web hosting service for user data stored in Arvados Keep" -package_go_binary services/datamanager arvados-data-manager \ - "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed" +package_go_binary sdk/go/crunchrunner crunchrunner \ + "Crunchrunner executes a command inside a container and uploads the output" package_go_binary services/arv-git-httpd arvados-git-httpd \ "Provide authenticated http access to Arvados-hosted git repositories" +package_go_binary services/crunch-dispatch-local crunch-dispatch-local \ + "Dispatch Crunch containers on the local system" +package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \ + "Dispatch Crunch containers to a SLURM cluster" +package_go_binary services/crunch-run crunch-run \ + "Supervise a single Crunch container" package_go_binary services/crunchstat crunchstat \ "Gather cpu/memory/network statistics of running Crunch jobs" -package_go_binary tools/keep-rsync keep-rsync \ - "Copy all data from one set of Keep servers to another" +package_go_binary services/datamanager arvados-data-manager \ + "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed" +package_go_binary services/keep-balance keep-balance \ + "Rebalance and garbage-collect data blocks stored in Arvados Keep" +package_go_binary services/keepproxy keepproxy \ + "Make a Keep cluster accessible to clients that are not on the LAN" +package_go_binary services/keepstore keepstore \ + "Keep storage daemon, accessible to clients on the LAN" +package_go_binary services/keep-web keep-web \ + "Static web hosting service for user data stored in Arvados Keep" package_go_binary tools/keep-block-check keep-block-check \ "Verify that all data from one set of Keep servers to another was copied" -package_go_binary sdk/go/crunchrunner crunchrunner \ - "Crunchrunner executes a command inside a container and uploads the output" +package_go_binary tools/keep-rsync keep-rsync \ + "Copy all data from one set of Keep servers to another" # The Python SDK # Please resist the temptation to add --no-python-fix-name to the fpm call here @@ -437,10 +448,12 @@ cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/sdk/cwl/build" fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3 +fpm_build lockfile "" "" python 0.12.2 --epoch 1 + # schema_salad. This is a python dependency of arvados-cwl-runner, # but we can't use the usual PYTHONPACKAGES way to build this package due to the -# intricacies of how version numbers get generated in setup.py: we need version -# 1.7.20160316203940. If we don't explicitly list that version with the -v +# intricacies of how version numbers get generated in setup.py: we need a specific version, +# e.g. 1.7.20160316203940. If we don't explicitly list that version with the -v # argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get # a package with version 1.7. That's because our gittagger hack is not being # picked up by self.distribution.get_version(), which is called from @@ -452,14 +465,18 @@ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curover # So we build this thing separately. # # Ward, 2016-03-17 -fpm_build schema_salad "" "" python 1.11.20160506154702 +fpm_build schema_salad "" "" python 1.18.20161005190847 --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" # And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh. # Ward, 2016-05-26 -fpm_build ruamel.yaml "" "" python "" --python-setup-py-arguments "--single-version-externally-managed" +fpm_build ruamel.yaml "" "" python 0.12.4 --python-setup-py-arguments "--single-version-externally-managed" + +# Dependency of cwltool. Fpm doesn't produce a package with the correct version +# number unless we build it explicitly +fpm_build cwltest "" "" python 1.0.20160907111242 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17 -fpm_build cwltool "" "" python 1.0.20160519182434 +fpm_build cwltool "" "" python 1.0.20161007181528 # FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25 fpm_build rdflib-jsonld "" "" python 0.3.0