Further changes to make pipeline running work reliably.

refs #3219

diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index ad85315b66615ab42fe6261fca3f9f3a8b606915..6a70fc30eb3672e5069d4c4a7771cfda5fcab17e 100644 (file)
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -4,14 +4,10 @@ FROM arvados/passenger
 MAINTAINER Tim Pierce <twp@curoverse.com>
 
 # Install postgres and apache.
-# Clone a git repository of Arvados source -- not used to build, but
-# will be used by the Commit model and anything else that needs to
-# check a git repo for crunch scripts.
 RUN apt-get update && \
     apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
                           supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
-                          libjson-perl cron && \
-    git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
+                          libjson-perl cron
 
 ADD munge.key /etc/munge/
 RUN chown munge:munge /etc/munge/munge.key && chmod 600 /etc/munge/munge.key
@@ -75,7 +71,7 @@ ADD update-gitolite.rb /usr/local/arvados/
 ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
 ADD ssh.sh /usr/local/bin/ssh.sh
 ADD generated/setup.sh /usr/local/bin/setup.sh
-ADD setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
+ADD generated/setup-gitolite.sh /usr/local/bin/setup-gitolite.sh
 ADD crunch-dispatch-run.sh /usr/local/bin/crunch-dispatch-run.sh
 ADD apache2_foreground.sh /etc/apache2/foreground.sh
 

diff --git a/docker/api/application.yml.in b/docker/api/application.yml.in
index f832b7f3d40ff6e3c62891814fed250f9a923f94..355c4e5f94be8aa4ceda83272daffd00433e311b 100644 (file)
--- a/docker/api/application.yml.in
+++ b/docker/api/application.yml.in
@@ -51,8 +51,10 @@ production:
   #   end << '172.16.0.23'
   # %>
   permit_create_collection_with_unsigned_manifest: true
-  git_repositories_dir: /var/cache/git
+  git_repositories_dir: /home/git/repositories
   crunch_job_wrapper: :slurm_immediate
+  action_mailer.raise_delivery_errors: false
+  action_mailer.perform_deliveries: false
 
 test:
   uuid_prefix: zzzzz

diff --git a/docker/api/setup-gitolite.sh b/docker/api/setup-gitolite.sh.in
similarity index 60%
rename from docker/api/setup-gitolite.sh
rename to docker/api/setup-gitolite.sh.in
index c55fdeea0ed2d714fb3b0c173768b89f1b5eded4..0363063f58968636e93f9e375e2166ad8f45f92b 100755 (executable)
--- a/docker/api/setup-gitolite.sh
+++ b/docker/api/setup-gitolite.sh.in
@@ -14,6 +14,14 @@ su - git -c "git clone git://github.com/sitaramc/gitolite"
 su - git -c "gitolite/install -ln ~/bin"
 su - git -c "PATH=/home/git/bin:$PATH gitolite setup -pk ~git/root-authorized_keys.pub"
 
+# Make sure the repositories are created in such a way that they are readable
+# by the api server
+sed -i 's/0077/0022/g' /home/git/.gitolite.rc
+
+# And make sure that the existing repos are equally readable, or the API server commit model will freak out...
+chmod 755 /home/git/repositories
+chmod +rx /home/git/repositories/*git -R
+
 # Now set up the gitolite repo(s) we use
 mkdir -p /usr/local/arvados/gitolite-tmp/
 # Make ssh store the host key
@@ -50,6 +58,20 @@ git add conf/gitolite.conf
 git commit -a -m 'git server setup'
 git push
 
+# Prepopulate the arvados.git repo with our source. Silly, but until we can check out from remote trees,
+# we need this to make the tutorials work.
+su - git -c "git clone --bare git://github.com/curoverse/arvados.git /home/git/repositories/arvados.git"
+
 echo "ARVADOS_API_HOST_INSECURE=yes" > /etc/cron.d/gitolite-update
-echo "*/5 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+echo "*/2 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+
+# Create/update the repos now
+. /etc/profile.d/rvm.sh
+export ARVADOS_API_HOST=api
+export ARVADOS_API_HOST_INSECURE=yes
+export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
+/usr/local/arvados/update-gitolite.rb production
+
+echo "PATH=/usr/bin:/bin:/sbin" > /etc/cron.d/arvados-repo-update
+echo "*/5 * * * * git cd ~git/repositories/arvados.git; git fetch https://github.com/curoverse/arvados.git master:master" >> /etc/cron.d/arvados-repo-update
 

diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index f1071094afd6fcf3679f7d2cbd9c8bce2f3c8e5b..c25a0d623166b799daa4a5cdd9072972972a051e 100755 (executable)
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -19,7 +19,7 @@ EOF
 
 # Arvados repository object
 all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
-repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados"}'`
+repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados","fetch_url":"git@api:arvados.git","push_url":"git@api:arvados.git"}'`
 echo "Arvados repository uuid is $repo_uuid"
 
 read -rd $'\000' newlink <<EOF; arv link create --link "$newlink" 

diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 172da98820e0558a1d5dd8fb80c352959244b6e0..223440ea740106f095a8f2b01afcddd98bb66813 100644 (file)
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -61,6 +61,7 @@ API_GENERATED = \
         api/generated/omniauth.rb \
         api/generated/application.yml \
         api/generated/setup.sh \
+        api/generated/setup-gitolite.sh \
         api/generated/slurm.conf \
         api/generated/superuser_token
 
@@ -72,6 +73,7 @@ API_GENERATED_IN = \
         api/omniauth.rb.in \
         api/application.yml.in \
         api/setup.sh.in \
+        api/setup-gitolite.sh.in \
         api/slurm.conf.in \
         api/superuser_token.in
 
@@ -145,6 +147,7 @@ api-image: passenger-image $(BUILD) $(API_DEPS)
        mkdir -p api/generated
        tar -czf api/generated/api.tar.gz -C build/services api
        chmod 755 api/generated/setup.sh
+       chmod 755 api/generated/setup-gitolite.sh
        $(DOCKER_BUILD) -t arvados/api api
        date >api-image