case "$TARGET" in
centos*)
- fpm_depends+=(git arvados-server)
+ fpm_depends+=(git)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++ arvados-server)
+ fpm_depends+=(git g++)
;;
esac
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
# The version of setuptools that comes with CentOS is way too old
RUN pip install --upgrade setuptools
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
-# Workbench depends on arvados-server for config manipulation
-ENV GOPATH /tmp
-RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
-
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
-set -x
# Build packages.
if docker run \
--rm \
fpm_build_virtualenv "cwltest" "cwltest"
rm -rf "$WORKSPACE/cwltest"
+calculate_go_package_version arvados_server_version cmd/arvados-server
+arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
+
# Build the API server package
test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
if [[ "$?" == "0" ]]; then
handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados API server - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
# Build the workbench server package
if [[ "$?" == "0" ]] ; then
(
set -e
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
cd "$WORKSPACE/apps/workbench"
# We need to bundle to be ready even when we build a package without vendor directory
mv /tmp/x /etc/arvados/config.yml
perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
- RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >/dev/null
- RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
+ RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >"$STDOUT_IF_DEBUG"
+ RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >"$STDOUT_IF_DEBUG"
# Remove generated configuration files so they don't go in the package.
rm -rf /etc/arvados/
handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
"$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
--description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0"
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
fi
fi
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
+--debug
+ Output debug information (default: false)
--rc
Optional Parameter to build Release Candidate
--build-version <version>
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,rc,target:,build-version: \
+ help,debug,upload,rc,target:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
TARGET=debian9
UPLOAD=0
RC=0
+DEBUG=
declare -a build_args=()
--target)
TARGET="$2"; shift
;;
+ --debug)
+ DEBUG=" --debug"
+ ;;
--upload)
UPLOAD=1
;;
title "Start build packages"
timer_reset
-$WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}"
+$WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}"$DEBUG
checkexit $? "build packages"
title "End of build packages (`timer`)"
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
- $WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}" --test-packages
+ $WORKSPACE/build/run-build-packages-one-target.sh "${build_args[@]}" --test-packages$DEBUG
else
echo "Skipping package upload, there were errors building the packages"
fi
fi
}
+calculate_go_package_version() {
+ # $__returnvar has the nameref attribute set, which means it is a reference
+ # to another variable that is passed in as the first argument to this function.
+ # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
+ local -n __returnvar="$1"; shift
+ local src_path="$1"; shift
+
+ mkdir -p "$GOPATH/src/git.curoverse.com"
+ ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
+ (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
+
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
+ local version="$(version_from_git)"
+ local timestamp="$(timestamp_from_git)"
+
+ # Update the version number and build a new package if the vendor
+ # bundle has changed, or the command imports anything from the
+ # Arvados SDK and the SDK has changed.
+ declare -a checkdirs=(vendor)
+ if grep -qr git.curoverse.com/arvados .; then
+ checkdirs+=(sdk/go lib)
+ fi
+ for dir in ${checkdirs[@]}; do
+ cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
+ ts="$(timestamp_from_git)"
+ if [[ "$ts" -gt "$timestamp" ]]; then
+ version=$(version_from_git)
+ timestamp="$ts"
+ fi
+ done
+
+ __returnvar="$version"
+}
+
# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
package_go_binary() {
local src_path="$1"; shift
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
debug_echo "package_go_binary $src_path as $prog"
local basename="${src_path##*/}"
-
- mkdir -p "$GOPATH/src/git.curoverse.com"
- ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
- (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v)
-
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path"
- local version="$(version_from_git)"
- local timestamp="$(timestamp_from_git)"
-
- # Update the version number and build a new package if the vendor
- # bundle has changed, or the command imports anything from the
- # Arvados SDK and the SDK has changed.
- declare -a checkdirs=(vendor)
- if grep -qr git.curoverse.com/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
- for dir in ${checkdirs[@]}; do
- cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
- ts="$(timestamp_from_git)"
- if [[ "$ts" -gt "$timestamp" ]]; then
- version=$(version_from_git)
- timestamp="$ts"
- fi
- done
+ calculate_go_package_version go_package_version $src_path
cd $WORKSPACE/packages/$TARGET
- test_package_presence $prog $version go
+ test_package_presence $prog $go_package_version go
if [[ "$?" != "0" ]]; then
return 1
fi
- go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
+ go get -ldflags "-X main.version=${go_package_version}" "git.curoverse.com/arvados.git/$src_path"
local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
fi
switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
- fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+ fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
default_iteration() {
test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
-test_package_presence() {
- local pkgname="$1"; shift
- local version="$1"; shift
- local pkgtype="$1"; shift
- local iteration="$1"; shift
- local arch="$1"; shift
+get_complete_package_name() {
+ # if the errexit flag is set, unset it until this function returns
+ # otherwise, the shift calls below will abort the program if optional arguments are not supplied
+ if [ -o errexit ]; then
+ set +e
+ trap 'set -e' RETURN
+ fi
+ # $__returnvar has the nameref attribute set, which means it is a reference
+ # to another variable that is passed in as the first argument to this function.
+ # see https://www.gnu.org/software/bash/manual/html_node/Shell-Parameters.html
+ local -n __returnvar="$1"; shift
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ "$iteration" == "" ]]; then
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ fi
- if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
- return 1
- fi
+ if [[ "$arch" == "" ]]; then
+ rpm_architecture="x86_64"
+ deb_architecture="amd64"
- if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ if [[ "$pkgtype" =~ ^(src)$ ]]; then
+ rpm_architecture="noarch"
+ deb_architecture="all"
fi
- if [[ "$arch" == "" ]]; then
+ # These python packages have binary components
+ if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
rpm_architecture="x86_64"
deb_architecture="amd64"
+ fi
+ else
+ rpm_architecture=$arch
+ deb_architecture=$arch
+ fi
- if [[ "$pkgtype" =~ ^(src)$ ]]; then
- rpm_architecture="noarch"
- deb_architecture="all"
- fi
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
+ if [[ "$FORMAT" == "rpm" ]]; then
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
+ fi
+ __returnvar=${complete_pkgname}
+}
- # These python packages have binary components
- if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+# Test if the package already exists, if not return 0, if it does return 1
+test_package_presence() {
+ local pkgname="$1"; shift
+ local version="$1"; shift
+ local pkgtype="$1"; shift
+ local iteration="$1"; shift
+ local arch="$1"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$pkgname" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ return 1
fi
- else
- rpm_architecture=$arch
- deb_architecture=$arch
fi
- if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
- else
- # rpm packages get iteration 1 if we don't supply one
- iteration=${iteration:-1}
- local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
- fi
+ local full_pkgname
+ get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch
# See if we can skip building the package, only if it already exists in the
# processed/ directory. If so, move it back to the packages directory to make
fi
repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
- echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ echo ${repo_pkg_list} |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+ echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${full_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
+ echo "Package $full_pkgname exists, not rebuilding!"
return 1
else
- echo "Package $complete_pkgname not found, building"
+ echo "Package $full_pkgname not found, building"
return 0
fi
else
centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
repo_pkg_list=$(curl -s -o - ${centos_repo})
- echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ echo ${repo_pkg_list} |grep -q ${full_pkgname}
if [ $? -eq 0 ]; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+ echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" ${centos_repo}${full_pkgname}
return 1
- elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then
- echo "Package $complete_pkgname exists, not rebuilding!"
+ elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
+ echo "Package $full_pkgname exists, not rebuilding!"
return 1
else
- echo "Package $complete_pkgname not found, building"
+ echo "Package $full_pkgname not found, building"
return 0
fi
fi
shift
if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$PACKAGE_NAME" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
return 0
+ fi
fi
local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
Site configuration file (default may be overridden by setting an ARVADOS_CONFIG environment variable) (default "/etc/arvados/config.yml")
-dump-config
write current configuration to stdout and exit
- -legacy-crunch-dispatch-slurm-config file
- Legacy crunch-dispatch-slurm configuration file (default "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml")
- -legacy-keepstore-config file
- Legacy keepstore configuration file (default "/etc/arvados/keepstore/keepstore.yml")
- -legacy-keepweb-config file
- Legacy keep-web configuration file (default "/etc/arvados/keep-web/keep-web.yml")
- -legacy-ws-config file
- Legacy arvados-ws configuration file (default "/etc/arvados/ws/ws.yml")
- -skip-legacy
- Don't load legacy config files
+[...]
-version
print version information and exit.
</code></pre>
</notextile>
-{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
-{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
-If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token. You can use the same one you used when you set up your Keepproxy server, or use the following command on the <strong>API server</strong> to create another. {% include 'install_rails_command' %}
-
-Install runit to supervise the Keep-web daemon. {% include 'install_runit' %}
-
-Set the cluster config file like the following:
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- SystemRootToken: "{{railsout}}"
- Services:
- Controller:
- ExternalURL: "https://<span class="userinput">uuid_prefix</span>.your.domain"
- Insecure: false
- WebDAV:
- InternalURLs:
- "http://keep_web_hostname_goes_here:9002/": {}
- WebDAVDownload:
- InternalURLs:
- "http://keep_web_hostname_goes_here:9002/": {}
- ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain/"
- Users:
- AnonymousUserToken: "xxxxxxxxxxxxxxxxxxxx"
- Collections:
- TrustAllContent: false
-</code></pre>
-</notextile>
-
-The basic command to start Keep-web in the service run script is:
-
-<notextile>
-<pre><code>exec sudo -u nobody keep-web
-</code></pre>
-</notextile>
-
-{% include 'notebox_begin' %}
-Please take into consideration that the config file should be world-readable.
-{% include 'notebox_end' %}
-
-Set @Users.AnonymousUserToken: ""@ (empty string) if you do not want to serve public data.
-
-Set @TLS.Insecure: true@ if your API server's TLS certificate is not signed by a recognized CA.
-
h3. Set up a reverse proxy with TLS support
The Keep-web service will be accessible from anywhere on the internet, so we recommend using TLS for transport encryption.
# Serve web content at @collections.uuid_prefix.your.domain@, but only for unauthenticated requests (public data and collection sharing links). Authenticated requests will always result in file downloads, using the @download@ name. For example, the Workbench "preview" button and the "view entire log file" link will invoke file downloads instead of displaying content in the browser window.
# In the special case where you know you are immune to XSS exploits, you can enable the "trust all content" mode in Keep-web and Workbench (setting @Collections.TrustAllContent: true@ on the config file). With this enabled, inline web content can be served from a single @collections@ host name; no wildcard DNS or certificate is needed. Do not do this without understanding the security implications described in the "Keep-web documentation":http://godoc.org/github.com/curoverse/arvados/services/keep-web.
-h3. Tell Workbench about the Keep-web service
+h2. Configure Keep-web
+
+{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
+{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
+If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token. You can use the same one you used when you set up your Keepproxy server, or use the following command on the <strong>API server</strong> to create another. {% include 'install_rails_command' %}
+
+Set the cluster config file like the following:
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Services:
+ Controller:
+ ExternalURL: "https://<span class="userinput">uuid_prefix</span>.your.domain"
+ WebDAV:
+ InternalURLs:
+ "http://keep_web_hostname_goes_here:9002/": {}
+ ExternalURL: "https://collections.<span class="userinput">uuid_prefix</span>.your.domain"
+ WebDAVDownload:
+ InternalURLs:
+ "http://keep_web_hostname_goes_here:9002/": {}
+ ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain"
+ Users:
+ AnonymousUserToken: "{{railsout}}"
+ Collections:
+ TrustAllContent: false
+ TLS:
+ Insecure: false
+</code></pre>
+</notextile>
+
+Set @Users.AnonymousUserToken: ""@ (empty string) if you do not want to serve public data.
+
+Set @TLS.Insecure: true@ if your API server's TLS certificate is not signed by a recognized CA.
Workbench has features like "download file from collection" and "show image" which work better if the content is served by Keep-web rather than Workbench itself. We recommend using the two different hostnames ("download" and "collections" above) for file downloads and inline content respectively.
-Add the following entry to your cluster configuration file (@/etc/arvados/config.yml@). This URL will be used for file downloads.
+The following entry on your cluster configuration file (@/etc/arvados/config.yml@) details the URL that will be used for file downloads.
<notextile>
<pre><code>Clusters:
- zzzzz:
+ <span class="userinput">uuid_prefix</span>:
Services:
WebDAVDownload:
- ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain/"
+ ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain"
</code></pre>
</notextile>
-Additionally, add *one* of the following entries to your Workbench cluster configuration file, depending on your DNS setup. This URL will be used to serve user content that can be displayed in the browser, like image previews and static HTML pages.
+Additionally, one of the following entries on your cluster configuration file (depending on your DNS setup) tells Workbench which URL will be used to serve user content that can be displayed in the browser, like image previews and static HTML pages.
<notextile>
<pre><code>Clusters:
- zzzzz:
+ <span class="userinput">uuid_prefix</span>:
Services:
WebDAV:
ExternalURL: "https://*--collections.<span class="userinput">uuid_prefix</span>.your.domain"
ExternalURL: "https://collections.<span class="userinput">uuid_prefix</span>.your.domain"
</code></pre>
</notextile>
+
+h2. Run Keep-web
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the keep-web service should already be set up. Start it and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart keep-web</span>
+~$ <span class="userinput">sudo systemctl status keep-web</span>
+● keep-web.service - Arvados Keep web gateway
+ Loaded: loaded (/lib/systemd/system/keep-web.service; enabled)
+ Active: active (running) since Sat 2019-08-10 10:33:21 UTC; 3 days ago
+ Docs: https://doc.arvados.org/
+ Main PID: 4242 (keep-web)
+ CGroup: /system.slice/keep-web.service
+ └─4242 /usr/bin/keep-web
+[...]
+</code></pre>
+</notextile>
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the Keep-web daemon. {% include 'install_runit' %}
+
+The basic command to start Keep-web in the service run script is:
+
+<notextile>
+<pre><code>exec keep-web
+</code></pre>
+</notextile>
+
# Cache parameters for WebDAV content serving:
# * TTL: Maximum time to cache manifests and permission checks.
# * UUIDTTL: Maximum time to cache collection state.
+ # * MaxBlockEntries: Maximum number of block cache entries.
# * MaxCollectionEntries: Maximum number of collection cache entries.
# * MaxCollectionBytes: Approximate memory limit for collection cache.
# * MaxPermissionEntries: Maximum number of permission cache entries.
WebDAVCache:
TTL: 300s
UUIDTTL: 5s
+ MaxBlockEntries: 4
MaxCollectionEntries: 1000
MaxCollectionBytes: 100000000
MaxPermissionEntries: 1000
# Cache parameters for WebDAV content serving:
# * TTL: Maximum time to cache manifests and permission checks.
# * UUIDTTL: Maximum time to cache collection state.
+ # * MaxBlockEntries: Maximum number of block cache entries.
# * MaxCollectionEntries: Maximum number of collection cache entries.
# * MaxCollectionBytes: Approximate memory limit for collection cache.
# * MaxPermissionEntries: Maximum number of permission cache entries.
WebDAVCache:
TTL: 300s
UUIDTTL: 5s
+ MaxBlockEntries: 4
MaxCollectionEntries: 1000
MaxCollectionBytes: 100000000
MaxPermissionEntries: 1000
type WebDAVCacheConfig struct {
TTL Duration
UUIDTTL Duration
+ MaxBlockEntries int
MaxCollectionEntries int
MaxCollectionBytes int64
MaxPermissionEntries int
case "$TARGET" in
centos*)
- fpm_depends+=(libcurl-devel postgresql-devel arvados-server)
+ fpm_depends+=(libcurl-devel postgresql-devel)
;;
debian* | ubuntu*)
- fpm_depends+=(libcurl-ssl-dev libpq-dev g++ arvados-server)
+ fpm_depends+=(libcurl-ssl-dev libpq-dev g++)
;;
esac
+++ /dev/null
-#!/usr/bin/env ruby
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-puts "**************************************
-The jobs API (crunch v1) is no longer supported. This is a stub
-script that exists only to assist in a smooth upgrade. You should
-remove crunch-dispatch.rb from your init configuration. This script
-will now sleep forever.
-**************************************
-"
-
-while true do
- sleep 10
-end
h.clientPool = arvadosclient.MakeClientPool()
keepclient.RefreshServiceDiscoveryOnSIGHUP()
+ keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
h.healthHandler = &health.Handler{
Token: h.Config.cluster.ManagementToken,
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
s.Config = cfg
}
+func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
+ cfg := newConfig(s.Config)
+ cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ h := handler{Config: cfg}
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ }
+ resp := httptest.NewRecorder()
+ h.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
+}
+
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
h := handler{Config: newConfig(s.Config)}
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
ExternalURL: "https://$localip:${services[sso]}"
Websocket:
ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
+ InternalURLs:
+ "http://localhost:${services[websockets]}": {}
GitSSH:
ExternalURL: "ssh://git@$localip:"
GitHTTP:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
+ InternalURLs:
+ "http://localhost:${services[keep-web]}/": {}
Composer:
ExternalURL: "http://$localip:${services[composer]}"
Controller:
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
"http://localhost:${services[controller]}": {}
+ RailsAPI:
+ InternalURLs:
+ "http://localhost:${services[api]}/": {}
+ Keepproxy:
+ ExternalURL: "https://$localip:${services[keepproxy-ssl]}"
+ InternalURLs:
+ "http://localhost:${services[keepproxy]}": {}
PostgreSQL:
ConnectionPool: 32 # max concurrent connections per arvados server daemon
Connection:
[keep-web]=9003
[keep-web-ssl]=9002
[keepproxy]=25100
+ [keepproxy-ssl]=25101
[keepstore0]=25107
[keepstore1]=25108
[ssh]=22
read -rd $'\000' keepservice <<EOF
{
"service_host":"$localip",
- "service_port":${services[keepproxy]},
- "service_ssl_flag":false,
+ "service_port":${services[keepproxy-ssl]},
+ "service_ssl_flag":true,
"service_type":"proxy"
}
EOF
server_name keep-web;
ssl_certificate "${server_cert}";
ssl_certificate_key "${server_cert_key}";
+ client_max_body_size 0;
location / {
proxy_pass http://keep-web;
proxy_set_header Host \$http_host;
}
}
+
+ upstream keepproxy {
+ server localhost:${services[keepproxy]};
+ }
+ server {
+ listen *:${services[keepproxy-ssl]} ssl default_server;
+ server_name keepproxy;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+ client_max_body_size 128M;
+ location / {
+ proxy_pass http://keepproxy;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+
}
EOF
projects / arvados.git / commitdiff