From 150f495e96c43f5d7e0df58352c5b20cc9e83ccd Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Tue, 15 Jul 2014 15:27:04 -0400 Subject: [PATCH 1/1] Add local git repository support. refs #3219 --- docker/api/Dockerfile | 8 +- docker/api/arvados-clients.yml.in | 5 + docker/api/setup-gitolite.sh | 55 ++++++++++ docker/api/setup.sh.in | 2 - docker/api/supervisor.conf | 12 +++ docker/api/update-gitolite.rb | 166 ++++++++++++++++++++++++++++++ docker/build_tools/Makefile | 2 + docker/compute/supervisor.conf | 3 + docker/workbench/Dockerfile | 2 +- 9 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 docker/api/arvados-clients.yml.in create mode 100755 docker/api/setup-gitolite.sh create mode 100755 docker/api/update-gitolite.rb diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index ce5755d7c8..ad85315b66 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -10,7 +10,7 @@ MAINTAINER Tim Pierce RUN apt-get update && \ apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \ supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \ - libjson-perl && \ + libjson-perl cron && \ git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git ADD munge.key /etc/munge/ @@ -66,10 +66,16 @@ RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid ADD keep_server_0.json /root/ ADD keep_server_1.json /root/ +# Set up update-gitolite.rb +RUN mkdir /usr/local/arvados/config -p +ADD generated/arvados-clients.yml /usr/local/arvados/config/ +ADD update-gitolite.rb /usr/local/arvados/ + # Supervisor. ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf ADD ssh.sh /usr/local/bin/ssh.sh ADD generated/setup.sh /usr/local/bin/setup.sh +ADD setup-gitolite.sh /usr/local/bin/setup-gitolite.sh ADD crunch-dispatch-run.sh /usr/local/bin/crunch-dispatch-run.sh ADD apache2_foreground.sh /etc/apache2/foreground.sh diff --git a/docker/api/arvados-clients.yml.in b/docker/api/arvados-clients.yml.in new file mode 100644 index 0000000000..babfc4e5c9 --- /dev/null +++ b/docker/api/arvados-clients.yml.in @@ -0,0 +1,5 @@ +production: + gitolite_url: 'git@api:gitolite-admin.git' + gitolite_tmp: 'gitolite-tmp' + arvados_api_host: 'api' + arvados_api_token: '@@API_SUPERUSER_SECRET@@' diff --git a/docker/api/setup-gitolite.sh b/docker/api/setup-gitolite.sh new file mode 100755 index 0000000000..c55fdeea0e --- /dev/null +++ b/docker/api/setup-gitolite.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +ssh-keygen -q -N '' -t rsa -f /root/.ssh/id_rsa + +useradd git +mkdir /home/git + +# Set up gitolite repository +cp ~root/.ssh/id_rsa.pub ~git/root-authorized_keys.pub +chown git:git /home/git -R +su - git -c "mkdir -p ~/bin" + +su - git -c "git clone git://github.com/sitaramc/gitolite" +su - git -c "gitolite/install -ln ~/bin" +su - git -c "PATH=/home/git/bin:$PATH gitolite setup -pk ~git/root-authorized_keys.pub" + +# Now set up the gitolite repo(s) we use +mkdir -p /usr/local/arvados/gitolite-tmp/ +# Make ssh store the host key +ssh -o "StrictHostKeyChecking no" git@api info +# Now check out the tree +git clone git@api:gitolite-admin.git /usr/local/arvados/gitolite-tmp/gitolite-admin/ +cd /usr/local/arvados/gitolite-tmp/gitolite-admin +mkdir keydir/arvados +mkdir conf/admin +mkdir conf/auto +echo " + +@arvados_git_user = arvados_git_user + +repo @all + RW+ = @arvados_git_user + +" > conf/admin/arvados.conf +echo ' +include "auto/*.conf" +include "admin/*.conf" +' >> conf/gitolite.conf + +#su - git -c "ssh-keygen -t rsa" +cp /root/.ssh/id_rsa.pub keydir/arvados/arvados_git_user.pub +# Replace the 'root' key with the user key, just in case +cp /root/.ssh/authorized_keys keydir/root-authorized_keys.pub +# But also make sure we have the root key installed so it can access all keys +git add keydir/root-authorized_keys.pub +git add keydir/arvados/arvados_git_user.pub +git add conf/admin/arvados.conf +git add keydir/arvados/ +git add conf/gitolite.conf +git commit -a -m 'git server setup' +git push + +echo "ARVADOS_API_HOST_INSECURE=yes" > /etc/cron.d/gitolite-update +echo "*/5 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update + diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in index fd4cf07652..f1071094af 100755 --- a/docker/api/setup.sh.in +++ b/docker/api/setup.sh.in @@ -9,7 +9,6 @@ export ARVADOS_API_HOST_INSECURE=yes export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@ # All users group - prefix=`arv --format=uuid user current | cut -d- -f1` read -rd $'\000' newgroup < + +# Default is development +production = ARGV[0] == "production" + +ENV["RAILS_ENV"] = "development" +ENV["RAILS_ENV"] = "production" if production + +DEBUG = 1 + +# load and merge in the environment-specific application config info +# if present, overriding base config parameters as specified +path = File.dirname(__FILE__) + '/config/arvados-clients.yml' +if File.exists?(path) then + cp_config = YAML.load_file(path)[ENV['RAILS_ENV']] +else + puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file" + exit 1 +end + +gitolite_url = cp_config['gitolite_url'] +gitolite_tmp = cp_config['gitolite_tmp'] + +gitolite_admin = File.join(File.expand_path(File.dirname(__FILE__)) + '/' + gitolite_tmp + '/gitolite-admin') + +ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host'] +ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token'] + +keys = '' + +seen = Hash.new + +def ensure_repo(name,permissions,user_keys,gitolite_admin) + tmp = '' + # Just in case... + name.gsub!(/[^a-z0-9]/i,'') + + keys = Hash.new() + + user_keys.each do |uuid,p| + p.each do |k| + next if k[:public_key].nil? + keys[uuid] = Array.new() if not keys.key?(uuid) + + key = k[:public_key] + # Handle putty-style ssh public keys + key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1') + key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1') + key.gsub!(/\n/,'') + key.strip + + keys[uuid].push(key) + end + end + + cf = gitolite_admin + '/conf/auto/' + name + '.conf' + + conf = "\nrepo #{name}\n" + + commit = false + + seen = {} + permissions.sort.each do |uuid,v| + conf += "\t#{v[:gitolite_permissions]}\t= #{uuid.to_s}\n" + + count = 0 + keys.include?(uuid) and keys[uuid].each do |v| + kf = gitolite_admin + '/keydir/arvados/' + uuid.to_s + "@#{count}.pub" + seen[kf] = true + if !File.exists?(kf) or IO::read(kf) != v then + commit = true + f = File.new(kf + ".tmp",'w') + f.write(v) + f.close() + # File.rename will overwrite the destination file if it exists + File.rename(kf + ".tmp",kf); + end + count += 1 + end + end + + if !File.exists?(cf) or IO::read(cf) != conf then + commit = true + f = File.new(cf + ".tmp",'w') + f.write(conf) + f.close() + # this is about as atomic as we can make the replacement of the file... + File.unlink(cf) if File.exists?(cf) + File.rename(cf + ".tmp",cf); + end + + return commit,seen +end + +begin + + pwd = Dir.pwd + # Get our local gitolite-admin repo up to snuff + if not File.exists?(File.dirname(__FILE__) + '/' + gitolite_tmp) then + Dir.mkdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp), 0700) + end + if not File.exists?(gitolite_admin) then + Dir.chdir(File.join(File.dirname(__FILE__) + '/' + gitolite_tmp)) + `git clone #{gitolite_url}` + else + Dir.chdir(gitolite_admin) + `git pull` + end + Dir.chdir(pwd) + + arv = Arvados.new( { :suppress_ssl_warnings => false } ) + + permissions = arv.repository.get_all_permissions + + repos = permissions[:repositories] + user_keys = permissions[:user_keys] + + @commit = false + + @seen = {} + + repos.each do |r| + next if r[:name].nil? + (@c,@s) = ensure_repo(r[:name],r[:user_permissions],user_keys,gitolite_admin) + @seen.merge!(@s) + @commit = true if @c + end + + # Clean up public key files that should not be present + Dir.glob(gitolite_admin + '/keydir/arvados/*.pub') do |key_file| + next if key_file =~ /arvados_git_user.pub$/ + next if @seen.has_key?(key_file) + puts "Extra file #{key_file}" + @commit = true + Dir.chdir(gitolite_admin) + key_file.gsub!(/^#{gitolite_admin}\//,'') + `git rm #{key_file}` + end + + if @commit then + message = "#{Time.now().to_s}: update from API" + Dir.chdir(gitolite_admin) + `git add --all` + `git commit -m '#{message}'` + `git push` + end + +rescue Exception => bang + puts "Error: " + bang.to_s + puts bang.backtrace.join("\n") + exit 1 +end + diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile index fd49b3cf50..172da98820 100644 --- a/docker/build_tools/Makefile +++ b/docker/build_tools/Makefile @@ -54,6 +54,7 @@ COMPUTE_GENERATED = compute/generated/setup.sh COMPUTE_GENERATED_IN = compute/setup.sh.in API_GENERATED = \ + api/generated/arvados-clients.yml \ api/generated/apache2_vhost \ api/generated/config_databases.sh \ api/generated/database.yml \ @@ -64,6 +65,7 @@ API_GENERATED = \ api/generated/superuser_token API_GENERATED_IN = \ + api/arvados-clients.yml.in \ api/apache2_vhost.in \ api/config_databases.sh.in \ api/database.yml.in \ diff --git a/docker/compute/supervisor.conf b/docker/compute/supervisor.conf index af081dfc00..f2cce3fe51 100644 --- a/docker/compute/supervisor.conf +++ b/docker/compute/supervisor.conf @@ -6,14 +6,17 @@ startsecs=0 [program:munge] user=root command=/etc/init.d/munge start +startsecs=0 [program:slurm] user=root command=/etc/init.d/slurm-llnl start +startsecs=0 [program:cron] user=root command=/etc/init.d/cron start +startsecs=0 [program:setup] user=root diff --git a/docker/workbench/Dockerfile b/docker/workbench/Dockerfile index 75e60c9cc2..97ed0135d6 100644 --- a/docker/workbench/Dockerfile +++ b/docker/workbench/Dockerfile @@ -4,7 +4,7 @@ FROM arvados/passenger MAINTAINER Ward Vandewege # We need graphviz for the provenance graphs -RUN apt-get update && apt-get install graphviz +RUN apt-get update && apt-get -qqy install graphviz # Update Arvados source RUN /bin/mkdir -p /usr/src/arvados/apps -- 2.30.2