16267: switch to `arvados-server install -type test` for installing
authorWard Vandewege <ward@curii.com>
Wed, 23 Sep 2020 23:46:21 +0000 (19:46 -0400)
committerWard Vandewege <ward@curii.com>
Thu, 24 Sep 2020 13:22:34 +0000 (09:22 -0400)
       dependencies.

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

31 files changed:
tools/arvbox/bin/arvbox
tools/arvbox/lib/arvbox/docker/Dockerfile.base
tools/arvbox/lib/arvbox/docker/Dockerfile.demo
tools/arvbox/lib/arvbox/docker/Dockerfile.dev
tools/arvbox/lib/arvbox/docker/api-setup.sh
tools/arvbox/lib/arvbox/docker/cluster-config.sh
tools/arvbox/lib/arvbox/docker/common.sh
tools/arvbox/lib/arvbox/docker/createusers.sh
tools/arvbox/lib/arvbox/docker/devenv.sh
tools/arvbox/lib/arvbox/docker/go-setup.sh
tools/arvbox/lib/arvbox/docker/keep-setup.sh
tools/arvbox/lib/arvbox/docker/runit/2
tools/arvbox/lib/arvbox/docker/runsu.sh
tools/arvbox/lib/arvbox/docker/service/api/run-service
tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
tools/arvbox/lib/arvbox/docker/service/certificate/run
tools/arvbox/lib/arvbox/docker/service/controller/run
tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
tools/arvbox/lib/arvbox/docker/service/keepproxy/run-service
tools/arvbox/lib/arvbox/docker/service/nginx/run
tools/arvbox/lib/arvbox/docker/service/postgres/run
tools/arvbox/lib/arvbox/docker/service/postgres/run-service
tools/arvbox/lib/arvbox/docker/service/ready/run-service
tools/arvbox/lib/arvbox/docker/service/vm/run
tools/arvbox/lib/arvbox/docker/service/vm/run-service
tools/arvbox/lib/arvbox/docker/service/websockets/run
tools/arvbox/lib/arvbox/docker/service/workbench/run
tools/arvbox/lib/arvbox/docker/service/workbench/run-service
tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
tools/arvbox/lib/arvbox/docker/waitforpostgres.sh

index 122e2bec7c5bb1790fe357f33ded44a369b0fc75..8fd77f1f2dd0937ed352f1e4023d6d717079c222 100755 (executable)
@@ -60,6 +60,8 @@ PIPCACHE="$ARVBOX_DATA/pip"
 NPMCACHE="$ARVBOX_DATA/npm"
 GOSTUFF="$ARVBOX_DATA/gopath"
 RLIBS="$ARVBOX_DATA/Rlibs"
+ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox"
+GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0"
 
 getip() {
     docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
@@ -78,7 +80,7 @@ gethost() {
 }
 
 getclusterid() {
-    docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/api_uuid_prefix
+    docker exec $ARVBOX_CONTAINER cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix
 }
 
 updateconf() {
@@ -96,7 +98,7 @@ EOF
 }
 
 listusers() {
-    docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+    docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py $ARVADOS_CONTAINER_PATH/cluster_config.yml $(getclusterid) list
 }
 
 wait_for_arvbox() {
@@ -129,9 +131,9 @@ docker_run_dev() {
            "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
            "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
            "--volume=$PG_DATA:/var/lib/postgresql:rw" \
-           "--volume=$VAR_DATA:/var/lib/arvados:rw" \
+           "--volume=$VAR_DATA:$ARVADOS_CONTAINER_PATH:rw" \
            "--volume=$PASSENGER:/var/lib/passenger:rw" \
-           "--volume=$GEMS:/var/lib/gems:rw" \
+           "--volume=$GEMS:$GEM_HOME:rw" \
            "--volume=$PIPCACHE:/var/lib/pip:rw" \
            "--volume=$NPMCACHE:/var/lib/npm:rw" \
            "--volume=$GOSTUFF:/var/lib/gopath:rw" \
@@ -220,7 +222,7 @@ run() {
         fi
 
         if ! (docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q) ; then
-            docker create -v /var/lib/postgresql -v /var/lib/arvados --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true
+            docker create -v /var/lib/postgresql -v $ARVADOS_CONTAINER_PATH --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true
         fi
 
         docker run \
@@ -264,11 +266,6 @@ run() {
                        $ARVBOX_CONTAINER \
                        /usr/local/lib/arvbox/runsu.sh \
                        /usr/local/lib/arvbox/waitforpostgres.sh
-
-                docker exec -ti \
-                       $ARVBOX_CONTAINER \
-                       /usr/local/lib/arvbox/runsu.sh \
-                       /var/lib/arvbox/service/api/run-service --only-setup
             fi
 
             interactive=""
@@ -281,12 +278,12 @@ run() {
                    -e COLUMNS=$(tput cols) \
                    -e TERM=$TERM \
                    -e WORKSPACE=/usr/src/arvados \
-                   -e GEM_HOME=/var/lib/gems \
-                   -e CONFIGSRC=/var/lib/arvados/run_tests \
+                   -e GEM_HOME=$GEM_HOME \
+                   -e CONFIGSRC=$ARVADOS_CONTAINER_PATH/run_tests \
                    $ARVBOX_CONTAINER \
                    /usr/local/lib/arvbox/runsu.sh \
                    /usr/src/arvados/build/run-tests.sh \
-                   --temp /var/lib/arvados/test \
+                   --temp $ARVADOS_CONTAINER_PATH/test \
                    $interactive \
                    "$@"
         elif [[ "$CONFIG" = devenv ]] ; then
@@ -299,15 +296,15 @@ run() {
                     --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \
                         arvados/arvbox-dev$TAG
             fi
-                exec docker exec --interactive --tty \
-                     -e LINES=$(tput lines) \
-                     -e COLUMNS=$(tput cols) \
-                     -e TERM=$TERM \
-                     -e "ARVBOX_HOME=$HOME" \
-                     -e "DISPLAY=$DISPLAY" \
-                     --workdir=$PWD \
+            exec docker exec --interactive --tty \
+                 -e LINES=$(tput lines) \
+                 -e COLUMNS=$(tput cols) \
+                 -e TERM=$TERM \
+                 -e "ARVBOX_HOME=$HOME" \
+                 -e "DISPLAY=$DISPLAY" \
+                 --workdir=$PWD \
                  ${ARVBOX_CONTAINER} \
-                     /usr/local/lib/arvbox/devenv.sh "$@"
+                 /usr/local/lib/arvbox/devenv.sh "$@"
         elif [[ "$CONFIG" =~ dev$ ]] ; then
             docker_run_dev \
                    --detach \
@@ -424,7 +421,7 @@ case "$subcmd" in
                -e LINES=$(tput lines) \
                -e COLUMNS=$(tput cols) \
                -e TERM=$TERM \
-               -e GEM_HOME=/var/lib/gems \
+               -e GEM_HOME=$GEM_HOME \
                $ARVBOX_CONTAINER /bin/bash
         ;;
 
@@ -433,14 +430,14 @@ case "$subcmd" in
                -e LINES=$(tput lines) \
                -e COLUMNS=$(tput cols) \
                -e TERM=$TERM \
-               -e GEM_HOME=/var/lib/gems \
+               -e GEM_HOME=$GEM_HOME \
                -u arvbox \
                -w /usr/src/arvados \
                $ARVBOX_CONTAINER /bin/bash --login
         ;;
 
     pipe)
-        exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash -
+        exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=$GEM_HOME /bin/bash -
         ;;
 
     stop)
@@ -587,24 +584,24 @@ case "$subcmd" in
         if test -n "$1" ; then
             CERT="$1"
         fi
-        docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT"
+        docker exec $ARVBOX_CONTAINER cat $ARVADOS_CONTAINER_PATH/root-cert.pem > "$CERT"
         echo "Certificate copied to $CERT"
         ;;
 
     psql)
-        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados'
+        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados'
         ;;
 
     checkpoint)
-        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql'
+        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > $ARVADOS_CONTAINER_PATH/checkpoint.sql'
         ;;
 
     restore)
-        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql'
+        exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=$ARVADOS_CONTAINER_PATH/checkpoint.sql'
         ;;
 
     hotreset)
-        exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - <<EOF
+        exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=$GEM_HOME /bin/bash - <<EOF
 sv stop api
 sv stop controller
 sv stop websockets
@@ -615,11 +612,11 @@ cd /usr/src/arvados/services/api
 export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
 export RAILS_ENV=development
 bundle exec rake db:drop
-rm /var/lib/arvados/api_database_setup
-rm /var/lib/arvados/superuser_token
-rm /var/lib/arvados/keep0-uuid
-rm /var/lib/arvados/keep1-uuid
-rm /var/lib/arvados/keepproxy-uuid
+rm $ARVADOS_CONTAINER_PATH/api_database_setup
+rm $ARVADOS_CONTAINER_PATH/superuser_token
+rm $ARVADOS_CONTAINER_PATH/keep0-uuid
+rm $ARVADOS_CONTAINER_PATH/keep1-uuid
+rm $ARVADOS_CONTAINER_PATH/keepproxy-uuid
 sv start api
 sv start controller
 sv start websockets
@@ -630,12 +627,12 @@ EOF
         ;;
 
     adduser)
-        docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml.override $(getclusterid) add $@
+        docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py $ARVADOS_CONTAINER_PATH/cluster_config.yml.override $(getclusterid) add $@
         docker exec $ARVBOX_CONTAINER sv restart controller
         ;;
 
     removeuser)
-        docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml.override $(getclusterid) remove $@
+        docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py $ARVADOS_CONTAINER_PATH/cluster_config.yml.override $(getclusterid) remove $@
         docker exec $ARVBOX_CONTAINER sv restart controller
         ;;
 
index eb52ca5a780e252873739c45a2e87fa667748024..9031de79bbaa051ec56e2274a100018fc6c79df3 100644 (file)
@@ -6,47 +6,36 @@ FROM debian:10
 
 ENV DEBIAN_FRONTEND noninteractive
 
+RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list
+
 RUN apt-get update && \
     apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
-    postgresql postgresql-contrib git build-essential runit curl libpq-dev \
-    libcurl4-openssl-dev libssl-dev zlib1g-dev libpcre3-dev libpam-dev \
-    openssh-server netcat-traditional \
-    graphviz bzip2 less sudo virtualenv \
-    fuse libfuse-dev \
-    pkg-config libattr1-dev \
-    libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
-    libjson-perl nginx gitolite3 lsof libreadline-dev \
-    apt-transport-https ca-certificates python3-yaml \
-    linkchecker python3-virtualenv python3-venv xvfb iceweasel \
-    libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \
-    libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \
-    python3-setuptools python3-pip default-jdk-headless bsdmainutils net-tools \
-    ruby ruby-dev bundler shellinabox  && \
-    apt-get clean
+    golang -t buster-backports
 
-ENV RUBYVERSION_MINOR 2.5
-ENV RUBYVERSION 2.5.1
-
-# Install Ruby from source
-# RUN cd /tmp && \
-#  curl -f http://cache.ruby-lang.org/pub/ruby/${RUBYVERSION_MINOR}/ruby-${RUBYVERSION}.tar.gz | tar -xzf - && \
-#  cd ruby-${RUBYVERSION} && \
-#  ./configure --disable-install-doc && \
-#  make && \
-#  make install && \
-#  cd /tmp && \
-#  rm -rf ruby-${RUBYVERSION}
+# The arvbox-specific dependencies are
+#  gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less
+RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \
+    build-essential ca-certificates git libpam0g-dev \
+    gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \
+    apt-get clean
 
-ENV GEM_HOME /var/lib/gems
-ENV PATH $PATH:/var/lib/gems/bin
+ENV GOPATH /var/lib/gopath
 
-ENV GOVERSION 1.15.2
+RUN cd /usr/src && \
+    git clone https://git.arvados.org/arvados.git && \
+    cd arvados && \
+    go mod download && \
+    cd cmd/arvados-server && \
+    go install && \
+    $GOPATH/bin/arvados-server install -type test
 
-# Install golang binary
-RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
-    tar -C /usr/local -xzf -
+RUN /etc/init.d/postgresql start && \
+    su postgres -c 'dropuser arvados' && \
+    su postgres -c 'createuser -s arvbox' && \
+    /etc/init.d/postgresql stop
 
-ENV PATH ${PATH}:/usr/local/go/bin
+ENV GEM_HOME /var/lib/arvados/lib/ruby/gems/2.5.0
+ENV PATH $PATH:$GEM_HOME/bin
 
 VOLUME /var/lib/docker
 VOLUME /var/log/nginx
@@ -62,35 +51,6 @@ RUN mkdir -p /etc/apt/sources.list.d && \
     apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \
     apt-get clean
 
-RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql
-
-ENV PJSVERSION=1.9.8
-# bitbucket is the origin, but downloads fail sometimes, so use our own mirror instead.
-#ENV PJSURL=https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2
-ENV PJSURL=http://cache.arvados.org/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2
-
-RUN set -e && \
- curl -L -f ${PJSURL} | tar -C /usr/local -xjf - && \
- ln -s ../phantomjs-${PJSVERSION}-linux-x86_64/bin/phantomjs /usr/local/bin
-
-ENV GDVERSION=v0.23.0
-ENV GDURL=https://github.com/mozilla/geckodriver/releases/download/$GDVERSION/geckodriver-$GDVERSION-linux64.tar.gz
-RUN set -e && curl -L -f ${GDURL} | tar -C /usr/local/bin -xzf - geckodriver
-
-ENV NODEVERSION v8.15.1
-
-# Install nodejs binary
-RUN curl -L -f https://nodejs.org/dist/${NODEVERSION}/node-${NODEVERSION}-linux-x64.tar.xz | tar -C /usr/local -xJf - && \
-    ln -s ../node-${NODEVERSION}-linux-x64/bin/node ../node-${NODEVERSION}-linux-x64/bin/npm /usr/local/bin
-
-ENV GRADLEVERSION 5.3.1
-
-RUN cd /tmp && \
-    curl -L -O https://services.gradle.org/distributions/gradle-${GRADLEVERSION}-bin.zip && \
-    unzip gradle-${GRADLEVERSION}-bin.zip -d /usr/local && \
-    ln -s ../gradle-${GRADLEVERSION}/bin/gradle /usr/local/bin && \
-    rm gradle-${GRADLEVERSION}-bin.zip
-
 # Set UTF-8 locale
 RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8
@@ -111,6 +71,11 @@ ADD gitolite.rc \
 
 ADD runit /etc/runit
 
+# arvbox mounts a docker volume at $ARVADOS_CONTAINER_PATH, make sure that that
+# doesn't overlap with the directory where `arvados-server install -type test`
+# put everything (/var/lib/arvados)
+ENV ARVADOS_CONTAINER_PATH /var/lib/arvados-arvbox
+
 # Start the supervisor.
 ENV SVDIR /etc/service
 STOPSIGNAL SIGINT
index ed728204fa171976ef316abd4060e8156b0da2ad..192b2a144c8ce9b652e311c2c67cdb139bdf9a75 100644 (file)
@@ -8,7 +8,6 @@ ARG composer_version=arvados-fork
 ARG workbench2_version=master
 
 RUN cd /usr/src && \
-    git clone --no-checkout https://git.arvados.org/arvados.git && \
     git -C arvados checkout ${arvados_version} && \
     git -C arvados pull && \
     git clone --no-checkout https://github.com/arvados/composer.git && \
@@ -19,11 +18,14 @@ RUN cd /usr/src && \
     git -C workbench2 pull && \
     chown -R 1000:1000 /usr/src
 
+# avoid rebuilding arvados-server, it's already been built as part of the base image
+RUN install $GOPATH/bin/arvados-server /usr/local/bin
+
 ADD service/ /var/lib/arvbox/service
 RUN ln -sf /var/lib/arvbox/service /etc
-RUN mkdir -p /var/lib/arvados
-RUN echo "production" > /var/lib/arvados/api_rails_env
-RUN echo "production" > /var/lib/arvados/workbench_rails_env
+RUN mkdir -p $ARVADOS_CONTAINER_PATH
+RUN echo "production" > $ARVADOS_CONTAINER_PATH/api_rails_env
+RUN echo "production" > $ARVADOS_CONTAINER_PATH/workbench_rails_env
 
 RUN /usr/local/lib/arvbox/createusers.sh
 
index c7621e387dbdb51882e08669016a456810da6f24..e9c296a190453e43f3093d4f3aa392be01ee1783 100644 (file)
@@ -7,11 +7,11 @@ ARG arvados_version
 
 ADD service/ /var/lib/arvbox/service
 RUN ln -sf /var/lib/arvbox/service /etc
-RUN mkdir -p /var/lib/arvados
-RUN echo "development" > /var/lib/arvados/api_rails_env
-RUN echo "development" > /var/lib/arvados/workbench_rails_env
+RUN mkdir -p $ARVADOS_CONTAINER_PATH
+RUN echo "development" > $ARVADOS_CONTAINER_PATH/api_rails_env
+RUN echo "development" > $ARVADOS_CONTAINER_PATH/workbench_rails_env
 
 RUN mkdir /etc/test-service && \
     ln -sf /var/lib/arvbox/service/postgres /etc/test-service && \
     ln -sf /var/lib/arvbox/service/certificate /etc/test-service
-RUN mkdir /etc/devenv-service
\ No newline at end of file
+RUN mkdir /etc/devenv-service
index 6a261bf4c55e17b0423071394d03213013a40a08..f20278a69c46a90560dfd9a8888681be076c01c5 100755 (executable)
@@ -11,27 +11,27 @@ set -ex -o pipefail
 
 cd /usr/src/arvados/services/api
 
-if test -s /var/lib/arvados/api_rails_env ; then
-  export RAILS_ENV=$(cat /var/lib/arvados/api_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then
+  export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env)
 else
   export RAILS_ENV=development
 fi
 
 set -u
 
-flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
 if test -a /usr/src/arvados/services/api/config/arvados_config.rb ; then
     rm -f config/application.yml config/database.yml
 else
-    uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
-    secret_token=$(cat /var/lib/arvados/api_secret_token)
-    blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
-    management_token=$(cat /var/lib/arvados/management_token)
-    database_pw=$(cat /var/lib/arvados/api_database_pw)
-    vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+    uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix)
+    secret_token=$(cat $ARVADOS_CONTAINER_PATH/api_secret_token)
+    blob_signing_key=$(cat $ARVADOS_CONTAINER_PATH/blob_signing_key)
+    management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token)
+    database_pw=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw)
+    vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid)
 
-cat >config/application.yml <<EOF
+    cat >config/application.yml <<EOF
 $RAILS_ENV:
   uuid_prefix: $uuid_prefix
   secret_token: $secret_token
@@ -51,18 +51,18 @@ $RAILS_ENV:
   ManagementToken: $management_token
 EOF
 
-(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
-sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config/database.yml
+    (cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
+    sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config/database.yml
 fi
 
-if ! test -f /var/lib/arvados/api_database_setup ; then
+if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
    bundle exec rake db:setup
-   touch /var/lib/arvados/api_database_setup
+   touch $ARVADOS_CONTAINER_PATH/api_database_setup
 fi
 
-if ! test -s /var/lib/arvados/superuser_token ; then
+if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
     superuser_tok=$(bundle exec ./script/create_superuser_token.rb)
-    echo "$superuser_tok" > /var/lib/arvados/superuser_token
+    echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
 fi
 
 rm -rf tmp
index bebf983b6bf07dc1af6748b4583fd8d17b43e86c..948eb00a559acfbb8f7ffaf48f8a79cbc9c8bfab 100755 (executable)
@@ -6,7 +6,9 @@
 exec 2>&1
 set -ex -o pipefail
 
-if [[ -s /etc/arvados/config.yml ]] && [[ /var/lib/arvados/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
+
+if [[ -s /etc/arvados/config.yml ]] && [[ $ARVADOS_CONTAINER_PATH/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then
    exit
 fi
 
@@ -14,58 +16,58 @@ fi
 
 set -u
 
-if ! test -s /var/lib/arvados/api_uuid_prefix ; then
-  ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > /var/lib/arvados/api_uuid_prefix
+if ! test -s $ARVADOS_CONTAINER_PATH/api_uuid_prefix ; then
+  ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > $ARVADOS_CONTAINER_PATH/api_uuid_prefix
 fi
-uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix)
 
-if ! test -s /var/lib/arvados/api_secret_token ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/api_secret_token
+if ! test -s $ARVADOS_CONTAINER_PATH/api_secret_token ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/api_secret_token
 fi
-secret_token=$(cat /var/lib/arvados/api_secret_token)
+secret_token=$(cat $ARVADOS_CONTAINER_PATH/api_secret_token)
 
-if ! test -s /var/lib/arvados/blob_signing_key ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/blob_signing_key
+if ! test -s $ARVADOS_CONTAINER_PATH/blob_signing_key ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/blob_signing_key
 fi
-blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+blob_signing_key=$(cat $ARVADOS_CONTAINER_PATH/blob_signing_key)
 
-if ! test -s /var/lib/arvados/management_token ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/management_token
+if ! test -s $ARVADOS_CONTAINER_PATH/management_token ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/management_token
 fi
-management_token=$(cat /var/lib/arvados/management_token)
+management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token)
 
-if ! test -s /var/lib/arvados/system_root_token ; then
-    ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/system_root_token
+if ! test -s $ARVADOS_CONTAINER_PATH/system_root_token ; then
+    ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/system_root_token
 fi
-system_root_token=$(cat /var/lib/arvados/system_root_token)
+system_root_token=$(cat $ARVADOS_CONTAINER_PATH/system_root_token)
 
-if ! test -s /var/lib/arvados/vm-uuid ; then
-    echo $uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]') > /var/lib/arvados/vm-uuid
+if ! test -s $ARVADOS_CONTAINER_PATH/vm-uuid ; then
+    echo $uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]') > $ARVADOS_CONTAINER_PATH/vm-uuid
 fi
-vm_uuid=$(cat /var/lib/arvados/vm-uuid)
+vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid)
 
-if ! test -f /var/lib/arvados/api_database_pw ; then
-    ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
+if ! test -f $ARVADOS_CONTAINER_PATH/api_database_pw ; then
+    ruby -e 'puts rand(2**128).to_s(36)' > $ARVADOS_CONTAINER_PATH/api_database_pw
 fi
-database_pw=$(cat /var/lib/arvados/api_database_pw)
+database_pw=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw)
 
 if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
     psql postgres -c "create user arvados with password '$database_pw'"
 fi
 psql postgres -c "ALTER USER arvados WITH SUPERUSER;"
 
-if ! test -s /var/lib/arvados/workbench_secret_token ; then
-  ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/workbench_secret_token
+if ! test -s $ARVADOS_CONTAINER_PATH/workbench_secret_token ; then
+  ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/workbench_secret_token
 fi
-workbench_secret_key_base=$(cat /var/lib/arvados/workbench_secret_token)
+workbench_secret_key_base=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
 
-if test -s /var/lib/arvados/api_rails_env ; then
-  database_env=$(cat /var/lib/arvados/api_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then
+  database_env=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env)
 else
   database_env=development
 fi
 
-cat >/var/lib/arvados/cluster_config.yml <<EOF
+cat >$ARVADOS_CONTAINER_PATH/cluster_config.yml <<EOF
 Clusters:
   ${uuid_prefix}:
     SystemRootToken: $system_root_token
@@ -143,41 +145,44 @@ Clusters:
       ArvadosDocsite: http://$localip:${services[doc]}/
     Git:
       GitCommand: /usr/share/gitolite3/gitolite-shell
-      GitoliteHome: /var/lib/arvados/git
-      Repositories: /var/lib/arvados/git/repositories
+      GitoliteHome: $ARVADOS_CONTAINER_PATH/git
+      Repositories: $ARVADOS_CONTAINER_PATH/git/repositories
     Volumes:
       ${uuid_prefix}-nyw5e-000000000000000:
         Driver: Directory
         DriverParameters:
-          Root: /var/lib/arvados/keep0
+          Root: $ARVADOS_CONTAINER_PATH/keep0
         AccessViaHosts:
           "http://localhost:${services[keepstore0]}": {}
       ${uuid_prefix}-nyw5e-111111111111111:
         Driver: Directory
         DriverParameters:
-          Root: /var/lib/arvados/keep1
+          Root: $ARVADOS_CONTAINER_PATH/keep1
         AccessViaHosts:
           "http://localhost:${services[keepstore1]}": {}
 EOF
 
-/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
+/usr/local/lib/arvbox/yml_override.py $ARVADOS_CONTAINER_PATH/cluster_config.yml
 
-cp /var/lib/arvados/cluster_config.yml /etc/arvados/config.yml
+cp $ARVADOS_CONTAINER_PATH/cluster_config.yml /etc/arvados/config.yml
 
+# Do not abort if certain optional files don't exist (e.g. cluster_config.yml.override)
+set +e
 chmod og-rw \
-      /var/lib/arvados/cluster_config.yml.override \
-      /var/lib/arvados/cluster_config.yml \
+      $ARVADOS_CONTAINER_PATH/cluster_config.yml.override \
+      $ARVADOS_CONTAINER_PATH/cluster_config.yml \
       /etc/arvados/config.yml \
-      /var/lib/arvados/api_secret_token \
-      /var/lib/arvados/blob_signing_key \
-      /var/lib/arvados/management_token \
-      /var/lib/arvados/system_root_token \
-      /var/lib/arvados/api_database_pw \
-      /var/lib/arvados/workbench_secret_token \
-      /var/lib/arvados/superuser_token \
-
-mkdir -p /var/lib/arvados/run_tests
-cat >/var/lib/arvados/run_tests/config.yml <<EOF
+      $ARVADOS_CONTAINER_PATH/api_secret_token \
+      $ARVADOS_CONTAINER_PATH/blob_signing_key \
+      $ARVADOS_CONTAINER_PATH/management_token \
+      $ARVADOS_CONTAINER_PATH/system_root_token \
+      $ARVADOS_CONTAINER_PATH/api_database_pw \
+      $ARVADOS_CONTAINER_PATH/workbench_secret_token \
+      $ARVADOS_CONTAINER_PATH/superuser_token \
+set -e
+
+mkdir -p $ARVADOS_CONTAINER_PATH/run_tests
+cat >$ARVADOS_CONTAINER_PATH/run_tests/config.yml <<EOF
 Clusters:
   zzzzz:
     PostgreSQL:
index 185467cf7de8d7d5b599f714bb4771090e05abb0..4bfe9dd513cdbd6264cd7b87edb2630384f2fd70 100644 (file)
@@ -3,12 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 export DEBIAN_FRONTEND=noninteractive
-export PATH=${PATH}:/usr/local/go/bin:/var/lib/gems/bin
-export GEM_HOME=/var/lib/gems
+export GEM_HOME=/var/lib/arvados/lib/ruby/gems/2.5.0
+export PATH=${PATH}:/usr/local/go/bin:$GEM_HOME/bin:/var/lib/arvados/bin
 export npm_config_cache=/var/lib/npm
 export npm_config_cache_min=Infinity
 export R_LIBS=/var/lib/Rlibs
 export HOME=$(getent passwd arvbox | cut -d: -f6)
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
 
 defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
 dockerip=$(/sbin/ip route | grep default | awk '{ print $3 }')
@@ -19,10 +20,10 @@ else
     localip=$containerip
 fi
 
-root_cert=/var/lib/arvados/root-cert.pem
-root_cert_key=/var/lib/arvados/root-cert.key
-server_cert=/var/lib/arvados/server-cert-${localip}.pem
-server_cert_key=/var/lib/arvados/server-cert-${localip}.key
+root_cert=$ARVADOS_CONTAINER_PATH/root-cert.pem
+root_cert_key=$ARVADOS_CONTAINER_PATH/root-cert.key
+server_cert=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem
+server_cert_key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key
 
 declare -A services
 services=(
@@ -62,22 +63,22 @@ run_bundler() {
         # The 'gem install bundler line below' is cf.
         # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html,
         # until we get bundler 2.7.10/3.0.0 or higher
-        gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
+        flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
         frozen=--frozen
     else
         frozen=""
     fi
-    # if ! test -x /var/lib/gems/bin/bundler ; then
+    # if ! test -x $GEM_HOME/bin/bundler ; then
     #  bundleversion=2.0.2
     #     bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true)
     #     if test -n "$bundlergem" ; then
-    #         flock /var/lib/gems/gems.lock gem install --verbose --local --no-document $bundlergem
+    #         flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem
     #     else
-    #         flock /var/lib/gems/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
+    #         flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
     #     fi
     # fi
-    if ! flock /var/lib/gems/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
-        flock /var/lib/gems/gems.lock bundler install --verbose --no-deployment $frozen "$@"
+    if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
+        flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
     fi
 }
 
index de1e7bba9606696d41fbc309f706a9a944888d49..cea23bc8158b5c5c03db2db8961a916e0e3e913f 100755 (executable)
@@ -5,16 +5,19 @@
 
 set -e -o pipefail
 
+export GEM_HOME=/var/lib/arvados/lib/ruby/gems/2.5.0
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
+
 if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
     HOSTUID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f4)
     HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5)
 
-    mkdir -p /var/lib/arvados/git /var/lib/gems \
+    mkdir -p $ARVADOS_CONTAINER_PATH/git $GEM_HOME \
           /var/lib/passenger /var/lib/gopath \
           /var/lib/pip /var/lib/npm
 
     if test -z "$ARVBOX_HOME" ; then
-       ARVBOX_HOME=/var/lib/arvados
+        ARVBOX_HOME=$ARVADOS_CONTAINER_PATH
     fi
 
     groupadd --gid $HOSTGID --non-unique arvbox
@@ -25,27 +28,25 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
             --groups docker \
             --shell /bin/bash \
             arvbox
-    useradd --home-dir /var/lib/arvados/git --uid $HOSTUID --gid $HOSTGID --non-unique git
+    useradd --home-dir $ARVADOS_CONTAINER_PATH/git --uid $HOSTUID --gid $HOSTGID --non-unique git
     useradd --groups docker crunch
 
     if [[ "$1" != --no-chown ]] ; then
-       chown arvbox:arvbox -R /usr/local /var/lib/arvados /var/lib/gems \
+        chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH $GEM_HOME \
               /var/lib/passenger /var/lib/postgresql \
               /var/lib/nginx /var/log/nginx /etc/ssl/private \
-              /var/lib/gopath /var/lib/pip /var/lib/npm
+              /var/lib/gopath /var/lib/pip /var/lib/npm \
+              /var/lib/arvados
     fi
 
-    mkdir -p /var/lib/gems/ruby
-    chown arvbox:arvbox -R /var/lib/gems/ruby
-
     mkdir -p /tmp/crunch0 /tmp/crunch1
     chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
 
     echo "arvbox    ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
 
     cat <<EOF > /etc/profile.d/paths.sh
-export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/go/bin:/var/lib/gems/bin:$(ls -d /usr/local/node-*)/bin
-export GEM_HOME=/var/lib/gems
+export PATH=/usr/local/bin:/usr/bin:/bin:$GEM_HOME/bin
+export GEM_HOME=/var/lib/arvados/lib/ruby/gems/2.5.0
 export npm_config_cache=/var/lib/npm
 export npm_config_cache_min=Infinity
 export R_LIBS=/var/lib/Rlibs
index 4df5463f1f06101b5dd82ac61281df805fa15721..b5c57f39fc959cba2cc2a72c3b8a313ea95cbf20 100755 (executable)
@@ -3,7 +3,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh --no-chown
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
+flock $ARVADOS_CONTAINER_PATH/createusers.lock /usr/local/lib/arvbox/createusers.sh --no-chown
 
 if [[ -n "$*" ]] ; then
     exec su --preserve-environment arvbox -c "$*"
index 9bee9104482525c5f2f7856dd9c39735915ab5d4..21be0ccd6fc93649e587596913f2bf35388d508a 100644 (file)
@@ -8,10 +8,14 @@ mkdir -p $GOPATH
 
 cd /usr/src/arvados
 if [[ $UID = 0 ]] ; then
-    /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go mod download
+  /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go mod download
+  if [[ ! -f /usr/local/bin/arvados-server ]]; then
     /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
+  fi
 else
-    flock /var/lib/gopath/gopath.lock go mod download
+  flock /var/lib/gopath/gopath.lock go mod download
+  if [[ ! -f /usr/local/bin/arvados-server ]]; then
     flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
+  fi
 fi
 install $GOPATH/bin/arvados-server /usr/local/bin
index 3bc3899b0b8e7c070f6d71f23dd604ade36950ef..657a9a2600abd5f2ec4f6e6e06d184f00e517c1d 100755 (executable)
@@ -17,11 +17,11 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-mkdir -p /var/lib/arvados/$1
+mkdir -p $ARVADOS_CONTAINER_PATH/$1
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
 
 set +e
 read -rd $'\000' keepservice <<EOF
@@ -34,25 +34,25 @@ read -rd $'\000' keepservice <<EOF
 EOF
 set -e
 
-if test -s /var/lib/arvados/$1-uuid ; then
-    keep_uuid=$(cat /var/lib/arvados/$1-uuid)
+if test -s $ARVADOS_CONTAINER_PATH/$1-uuid ; then
+    keep_uuid=$(cat $ARVADOS_CONTAINER_PATH/$1-uuid)
     arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
 else
     UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
-    echo $UUID > /var/lib/arvados/$1-uuid
+    echo $UUID > $ARVADOS_CONTAINER_PATH/$1-uuid
 fi
 
-management_token=$(cat /var/lib/arvados/management_token)
+management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token)
 
 set +e
 sv hup /var/lib/arvbox/service/keepproxy
 
-cat >/var/lib/arvados/$1.yml <<EOF
+cat >$ARVADOS_CONTAINER_PATH/$1.yml <<EOF
 Listen: "localhost:$2"
-BlobSigningKeyFile: /var/lib/arvados/blob_signing_key
-SystemAuthTokenFile: /var/lib/arvados/superuser_token
+BlobSigningKeyFile: $ARVADOS_CONTAINER_PATH/blob_signing_key
+SystemAuthTokenFile: $ARVADOS_CONTAINER_PATH/superuser_token
 ManagementToken: $management_token
 MaxBuffers: 20
 EOF
 
-exec /usr/local/bin/keepstore -config=/var/lib/arvados/$1.yml
+exec /usr/local/bin/keepstore -config=$ARVADOS_CONTAINER_PATH/$1.yml
index 5812f3d8b0cea307b793016156b8fa73b3909224..eccf62553eaac49e61285a16113cd78492f6ebf3 100755 (executable)
@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-PATH=/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/X11R6/bin
+PATH=/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin
 
 echo
 echo "Arvados-in-a-box starting"
index 88d832f0e837351ab79b5571163bd14a900e2429..674b15775564c5a4fcedffd01c50b8a9aa57b4b5 100755 (executable)
@@ -6,9 +6,11 @@
 HOSTUID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f4)
 HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5)
 
-flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
 
-export HOME=/var/lib/arvados
+flock $ARVADOS_CONTAINER_PATH/createusers.lock /usr/local/lib/arvbox/createusers.sh
+
+export HOME=$ARVADOS_CONTAINER_PATH
 
 chown arvbox /dev/stderr
 
index f052b5d636cf6095ce12b004d40ec87d4fd2812c..7df7b2820bf6a0b83bcb538ecb418e183e68afa8 100755 (executable)
@@ -10,8 +10,8 @@ set -ex -o pipefail
 
 cd /usr/src/arvados/services/api
 
-if test -s /var/lib/arvados/api_rails_env ; then
-  export RAILS_ENV=$(cat /var/lib/arvados/api_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then
+  export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env)
 else
   export RAILS_ENV=development
 fi
@@ -24,7 +24,7 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-flock /var/lib/arvados/api.lock /usr/local/lib/arvbox/api-setup.sh
+flock $ARVADOS_CONTAINER_PATH/api.lock /usr/local/lib/arvbox/api-setup.sh
 
 set +u
 if test "$1" = "--only-setup" ; then
index 5f71e5ab28a5d70a3e7ab5fa642f6b18d9c38a45..b369ff6228d0a90d9c1ef64968dde2b89d1ee7ff 100755 (executable)
@@ -18,7 +18,7 @@ fi
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export PATH="$PATH:/var/lib/arvados/git/bin"
+export PATH="$PATH:$ARVADOS_CONTAINER_PATH/git/bin"
 cd ~git
 
 exec /usr/local/bin/arv-git-httpd
index 6443b01793dd61aaa56e717aff71ec839a26c2c3..2536981a7aacd42dfca48694feb4155dc99f52b6 100755 (executable)
@@ -8,9 +8,9 @@ set -ex -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
-uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix)
 
 if ! openssl verify -CAfile $root_cert $root_cert ; then
     # req           signing request sub-command
@@ -74,13 +74,13 @@ if ! openssl verify -CAfile $root_cert $server_cert ; then
            -extensions x509_ext \
            -config <(cat /etc/ssl/openssl.cnf \
                          <(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
-            -out /var/lib/arvados/server-cert-${localip}.csr \
+            -out $ARVADOS_CONTAINER_PATH/server-cert-${localip}.csr \
             -keyout $server_cert_key \
             -days 365
 
     openssl x509 \
            -req \
-           -in /var/lib/arvados/server-cert-${localip}.csr \
+           -in $ARVADOS_CONTAINER_PATH/server-cert-${localip}.csr \
            -CA $root_cert \
            -CAkey $root_cert_key \
            -out $server_cert \
index 588e9d2dad216faffa9e96686977507b3bfe2eb8..e495e222e171cd784e581bb42e5af37bb22b842f 100755 (executable)
@@ -15,6 +15,6 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
 exec /usr/local/bin/arvados-controller
index 6e80d30ab9570fb8a26fcaeb8eee1c1f8e2ff8a0..821afdce50b1fac246071dabc7e5fd507b201c84 100755 (executable)
@@ -25,6 +25,6 @@ chmod +x /usr/local/bin/crunch-run.sh
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
 
 exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
index 6055efc4791e93978ac806f2f3111d7e15c758bb..e91386b6772a6e29184d53222c31c399e142f418 100755 (executable)
@@ -8,16 +8,16 @@ set -eux -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
-mkdir -p /var/lib/arvados/git
+mkdir -p $ARVADOS_CONTAINER_PATH/git
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
 
 export USER=git
 export USERNAME=git
 export LOGNAME=git
-export HOME=/var/lib/arvados/git
+export HOME=$ARVADOS_CONTAINER_PATH/git
 
 cd ~arvbox
 
@@ -33,7 +33,7 @@ if test -s ~arvbox/.ssh/known_hosts ; then
     ssh-keygen -f ".ssh/known_hosts" -R localhost
 fi
 
-if ! test -f /var/lib/arvados/gitolite-setup ; then
+if ! test -f $ARVADOS_CONTAINER_PATH/gitolite-setup ; then
     cd ~git
 
     # Do a no-op login to populate known_hosts
@@ -57,7 +57,7 @@ if ! test -f /var/lib/arvados/gitolite-setup ; then
     git config push.default simple
     git push
 
-    touch /var/lib/arvados/gitolite-setup
+    touch $ARVADOS_CONTAINER_PATH/gitolite-setup
 else
     # Do a no-op login to populate known_hosts
     # with the hostkey, so it won't try to ask
@@ -68,14 +68,14 @@ fi
 
 prefix=$(arv --format=uuid user current | cut -d- -f1)
 
-if ! test -s /var/lib/arvados/arvados-git-uuid ; then
+if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-uuid ; then
     repo_uuid=$(arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}")
-    echo $repo_uuid > /var/lib/arvados/arvados-git-uuid
+    echo $repo_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-uuid
 fi
 
-repo_uuid=$(cat /var/lib/arvados/arvados-git-uuid)
+repo_uuid=$(cat $ARVADOS_CONTAINER_PATH/arvados-git-uuid)
 
-if ! test -s /var/lib/arvados/arvados-git-link-uuid ; then
+if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid ; then
     all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
 
     set +e
@@ -89,19 +89,19 @@ if ! test -s /var/lib/arvados/arvados-git-link-uuid ; then
 EOF
     set -e
     link_uuid=$(arv --format=uuid link create --link "$newlink")
-    echo $link_uuid > /var/lib/arvados/arvados-git-link-uuid
+    echo $link_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid
 fi
 
-if ! test -d /var/lib/arvados/git/repositories/$repo_uuid.git ; then
-    git clone --bare /usr/src/arvados /var/lib/arvados/git/repositories/$repo_uuid.git
+if ! test -d $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git ; then
+    git clone --bare /usr/src/arvados $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git
 else
-    git --git-dir=/var/lib/arvados/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados master:master
+    git --git-dir=$ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados master:master
 fi
 
 cd /usr/src/arvados/services/api
 
-if test -s /var/lib/arvados/api_rails_env ; then
-  RAILS_ENV=$(cat /var/lib/arvados/api_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then
+  RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env)
 else
   RAILS_ENV=development
 fi
@@ -110,8 +110,8 @@ git_user_key=$(cat ~git/.ssh/id_rsa.pub)
 
 cat > config/arvados-clients.yml <<EOF
 $RAILS_ENV:
-  gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
-  gitolite_tmp: /var/lib/arvados/git
+  gitolite_url: $ARVADOS_CONTAINER_PATH/git/repositories/gitolite-admin.git
+  gitolite_tmp: $ARVADOS_CONTAINER_PATH/git
   arvados_api_host: $localip:${services[controller-ssl]}
   arvados_api_token: "$ARVADOS_API_TOKEN"
   arvados_api_host_insecure: false
index d093fbc885ad84fb33e626ff1a7f6ae2ca2dcd32..cf5ccd724b885ee84233b03e0259ef9c5906e8cc 100755 (executable)
@@ -19,7 +19,7 @@ fi
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
 
 set +e
 read -rd $'\000' keepservice <<EOF
@@ -32,12 +32,12 @@ read -rd $'\000' keepservice <<EOF
 EOF
 set -e
 
-if test -s /var/lib/arvados/keepproxy-uuid ; then
-    keep_uuid=$(cat /var/lib/arvados/keepproxy-uuid)
+if test -s $ARVADOS_CONTAINER_PATH/keepproxy-uuid ; then
+    keep_uuid=$(cat $ARVADOS_CONTAINER_PATH/keepproxy-uuid)
     arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
 else
     UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
-    echo $UUID > /var/lib/arvados/keepproxy-uuid
+    echo $UUID > $ARVADOS_CONTAINER_PATH/keepproxy-uuid
 fi
 
 exec /usr/local/bin/keepproxy
index cfb7788defc080986bb91fa0e5f1e9b871b27c5f..82db92137053f83f0282f24211e63824a7bb4a0c 100755 (executable)
@@ -21,9 +21,9 @@ fi
 
 openssl verify -CAfile $root_cert $server_cert
 
-cat <<EOF >/var/lib/arvados/nginx.conf
+cat <<EOF >$ARVADOS_CONTAINER_PATH/nginx.conf
 worker_processes auto;
-pid /var/lib/arvados/nginx.pid;
+pid $ARVADOS_CONTAINER_PATH/nginx.pid;
 
 error_log stderr;
 daemon off;
@@ -235,4 +235,4 @@ server {
 
 EOF
 
-exec nginx -c /var/lib/arvados/nginx.conf
+exec nginx -c $ARVADOS_CONTAINER_PATH/nginx.conf
index 3ef78ee45575676dc881059efd60cc57bd64cbd9..d8abc4d89d8b2037bdd240d0e3577b77e232bfc1 100755 (executable)
@@ -3,7 +3,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh
+export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
+flock $ARVADOS_CONTAINER_PATH/createusers.lock /usr/local/lib/arvbox/createusers.sh
 
 make-ssl-cert generate-default-snakeoil --force-overwrite
 
index f2377a0c2dd7a544cb0a5cf87318bada12b045e1..3569fd31264b2dfd849d653dcda065faa8bbf644 100755 (executable)
@@ -10,7 +10,6 @@ PGVERSION=11
 
 if ! test -d /var/lib/postgresql/$PGVERSION/main ; then
     /usr/lib/postgresql/$PGVERSION/bin/initdb --locale=en_US.UTF-8 -D /var/lib/postgresql/$PGVERSION/main
-    sh -c "while ! (psql postgres -c'\du' | grep '^ arvbox ') >/dev/null ; do createuser -s arvbox ; sleep 1 ; done" &
 fi
 mkdir -p /var/run/postgresql/$PGVERSION-main.pg_stat_tmp
 
index 21cb7d48c64060d0edf08ca135e8d7f4fa92fd4e..b29dafed70b7696aeaee8657e4f4206c8a7b3d79 100755 (executable)
@@ -49,9 +49,9 @@ export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
 
 vm_ok=0
-if test -s /var/lib/arvados/vm-uuid -a -s /var/lib/arvados/superuser_token; then
-    vm_uuid=$(cat /var/lib/arvados/vm-uuid)
-    export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+if test -s $ARVADOS_CONTAINER_PATH/vm-uuid -a -s $ARVADOS_CONTAINER_PATH/superuser_token; then
+    vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid)
+    export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
     if (which arv && arv virtual_machine get --uuid $vm_uuid) >/dev/null 2>/dev/null ; then
         vm_ok=1
     fi
@@ -63,7 +63,7 @@ fi
 
 if ! [[ -z "$waiting" ]] ; then
     if ps x | grep -v grep | grep "bundle install" > /dev/null; then
-        gemcount=$(ls /var/lib/gems/ruby/2.1.0/gems 2>/dev/null | wc -l)
+        gemcount=$(ls $GEM_HOME/gems 2>/dev/null | wc -l)
 
         gemlockcount=0
         for l in /usr/src/arvados/services/api/Gemfile.lock \
index ee210e35d8600d8cf90a01acf91f7624cb23c47c..4ea11aadcd85808dec396a87dfd575aacb07af56 100755 (executable)
@@ -16,8 +16,8 @@ cd /usr/src/arvados/services/login-sync
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
-export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
+export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid)
 
 while true ; do
       arvados-login-sync
index 932ba59818ba2fb604d4b11e744ffba37b622f78..5369af31d0c086cd986e2b9de877b5e196439d28 100755 (executable)
@@ -21,8 +21,8 @@ set -u
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
 export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
-export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat /var/lib/arvados/vm-uuid)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
+export ARVADOS_VIRTUAL_MACHINE_UUID=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid)
 
 set +e
 read -rd $'\000' vm <<EOF
index efa2e08a7a7f34c3a04ee4c213931ed37a4f65ab..f962c3e8f0004db016cec28240f7b5eb7285ed82 100755 (executable)
@@ -15,6 +15,6 @@ if test "$1" = "--only-deps" ; then
     exit
 fi
 
-/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
 exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws
index e163493781f1a16531dc7bb355137aed941843fa..b8a28fa762379d6735b1185e60296cb463c73b35 100755 (executable)
@@ -15,8 +15,8 @@ rm -rf tmp
 mkdir tmp
 chown arvbox:arvbox tmp
 
-if test -s /var/lib/arvados/workbench_rails_env ; then
-  export RAILS_ENV=$(cat /var/lib/arvados/workbench_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/workbench_rails_env ; then
+  export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/workbench_rails_env)
 else
   export RAILS_ENV=development
 fi
@@ -24,7 +24,7 @@ fi
 if test "$1" != "--only-deps" ; then
     openssl verify -CAfile $root_cert $server_cert
     exec bundle exec passenger start --port=${services[workbench]} \
-        --ssl --ssl-certificate=/var/lib/arvados/server-cert-${localip}.pem \
-        --ssl-certificate-key=/var/lib/arvados/server-cert-${localip}.key \
+        --ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \
+        --ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \
          --user arvbox
 fi
index 06742cf82e204649336ff3d21ce676d0bb7a2dd4..51b9420eebf542ecf0596e6dea6b1801a4966a11 100755 (executable)
@@ -10,8 +10,8 @@ set -ex -o pipefail
 
 cd /usr/src/arvados/apps/workbench
 
-if test -s /var/lib/arvados/workbench_rails_env ; then
-  export RAILS_ENV=$(cat /var/lib/arvados/workbench_rails_env)
+if test -s $ARVADOS_CONTAINER_PATH/workbench_rails_env ; then
+  export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/workbench_rails_env)
 else
   export RAILS_ENV=development
 fi
@@ -35,7 +35,7 @@ fi
 
 set -u
 
-secret_token=$(cat /var/lib/arvados/workbench_secret_token)
+secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
 
 if test -a /usr/src/arvados/apps/workbench/config/arvados_config.rb ; then
     rm -f config/application.yml
index e3fbd22c4575c0e5b69ac8debceef81d8b2ee19b..8c3c49efd65dced676317333f0fa755280d9bde2 100755 (executable)
@@ -27,7 +27,7 @@ cat <<EOF > /usr/src/workbench2/public/config.json
 EOF
 
 export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
 
 url_prefix="https://$localip:${services[workbench2-ssl]}/"
 
index 6bda618ab899e2a8ca1a429bf319f82263995c49..9b2eb69f9e97d53c3de952ac14b65f6fbf72ae08 100755 (executable)
@@ -9,6 +9,6 @@ while ! psql postgres -c\\du >/dev/null 2>/dev/null ; do
     sleep 1
 done
 
-while ! test -s /var/lib/arvados/server-cert-${localip}.pem ; do
+while ! test -s $ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem ; do
     sleep 1
 done