From: Tim Pierce Date: Thu, 13 Mar 2014 21:58:57 +0000 (-0400) Subject: Merge branch 'master' into 2221-complete-docker X-Git-Tag: 1.1.0~2690^2~64^2~17 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/76f9268395068d47fda7f34426bc83d603c834c7?hp=113d3f94224ca507ff6341ab61ac98429cad711e Merge branch 'master' into 2221-complete-docker --- diff --git a/.gitignore b/.gitignore index 4d6cc39a23..f27930edc8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,8 @@ docker/*/generated/* docker/config.yml doc/_site/* doc/.site/* -doc/sdk/python/arvados \ No newline at end of file +doc/sdk/python/arvados +sdk/perl/MYMETA.* +sdk/perl/Makefile +sdk/perl/blib/* +sdk/perl/pm_to_blib diff --git a/doc/install/client.html.textile.liquid b/doc/install/client.html.textile.liquid index b395d060f5..2c3b6eb48e 100644 --- a/doc/install/client.html.textile.liquid +++ b/doc/install/client.html.textile.liquid @@ -34,3 +34,17 @@ The arvados package includes the Ruby client library module. The arvados-cli pac {% include 'notebox_end' %} notextile. 
$ sudo gem install arvados arvados-cli
+ +h3. Perl + +{% include 'notebox_begin' %} +The Perl client library includes the @Arvados.pm@ module and submodules. +{% include 'notebox_end' %} + + +
$ cd arvados/sdk/perl
+$ perl Makefile.PL
+$ sudo make install
+
+ + diff --git a/doc/user/topics/tutorial-job-debug.html.textile.liquid b/doc/user/topics/tutorial-job-debug.html.textile.liquid index 0974e51697..a28a793d47 100644 --- a/doc/user/topics/tutorial-job-debug.html.textile.liquid +++ b/doc/user/topics/tutorial-job-debug.html.textile.liquid @@ -105,7 +105,7 @@ EOF 2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 check slurm allocation 2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 node localhost - 1 slots 2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start -2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world.py +2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world-fixed.py 2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_version /home/you/you 2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_parameters {} 2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 runtime_constraints {"max_tasks_per_node":0} @@ -122,11 +122,13 @@ EOF 2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 Freeze not implemented 2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 collate 2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 output 576c44d762ba241b0a674aa43152b52a+53 +WARNING:root:API lookup failed for collection 576c44d762ba241b0a674aa43152b52a+53 (: ) 2013-12-12_21:57:03 qr1hi-8i9sb-79260ykfew5trzl 31578 finish -2013-12-12_21:57:04 qr1hi-8i9sb-79260ykfew5trzl 31578 meta key is 9f937693334d0c9234ccc1f808ee7117+1761 +(The WARNING issued near the end of the script may be safely ignored here; it is the Arvados SDK letting you know that it could not find a collection named @576c44d762ba241b0a674aa43152b52a+53@ and that it is going to try looking up a block by that name instead.) + The job succeeded, with output in Keep object @576c44d762ba241b0a674aa43152b52a+53@. Let's look at our output: diff --git a/docker/Makefile b/docker/Makefile index c6f3dd7d12..d949db059e 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -33,7 +33,7 @@ WAREHOUSE_DEPS = warehouse/Dockerfile \ SSO_DEPS = sso/passenger.conf $(SSO_GENERATED) -BASE_GENERATED = base/generated +BASE_GENERATED = base/generated/arvados.tar.gz API_GENERATED = \ api/generated/apache2_vhost \ @@ -79,6 +79,10 @@ SSO_GENERATED_IN = \ $(BASE_GENERATED): config.yml ./config.rb + mkdir -p base/generated + tar -c -z -f base/generated/arvados.tar.gz -C .. . \ + --exclude=services/api/log/* --exclude=docker/* + $(API_GENERATED): config.yml $(API_GENERATED_IN) ./config.rb @@ -102,7 +106,7 @@ DOCKER_BUILD = docker build -q api-image: passenger-image $(API_DEPS) mkdir -p api/generated - tar -c -z -f api/generated/api.tar.gz -C ../services api + tar -c -z -f api/generated/api.tar.gz -C ../services api --exclude=api/log/* $(DOCKER_BUILD) -t arvados/api api echo -n "Built at $(date)" > api-image diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index d7fe554e92..6f3b792f58 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -27,7 +27,8 @@ ADD generated/apache2_vhost /etc/apache2/sites-available/arvados ENV RAILS_ENV production ADD generated/config_databases.sh /tmp/config_databases.sh ADD generated/superuser_token /tmp/superuser_token -RUN sh /tmp/config_databases.sh && \ +RUN bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \ + sh /tmp/config_databases.sh && \ rm /tmp/config_databases.sh && \ /etc/init.d/postgresql start && \ cd /usr/src/arvados/services/api && \ diff --git a/docker/api/production.rb.in b/docker/api/production.rb.in index 8d52babc5f..967d185f47 100644 --- a/docker/api/production.rb.in +++ b/docker/api/production.rb.in @@ -80,7 +80,7 @@ Server::Application.configure do # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5'] require 'net/http' config.compute_node_nameservers = [ '@@ARVADOS_DNS_SERVER@@' ] - + config.compute_node_domain = false config.uuid_prefix = '@@API_HOSTNAME@@' # Authentication stub: hard code pre-approved API tokens. diff --git a/docker/arvdock b/docker/arvdock index ed8da8bead..b2fa4b2cf5 100755 --- a/docker/arvdock +++ b/docker/arvdock @@ -34,7 +34,7 @@ function start_container { fi if [[ "$2" != '' ]]; then local name="$2" - args="$args -name $name" + args="$args --name $name" fi if [[ "$3" != '' ]]; then local volume="$3" @@ -42,7 +42,7 @@ function start_container { fi if [[ "$4" != '' ]]; then local link="$4" - args="$args -link $link" + args="$args --link $link" fi local image=$5 diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index de90a09394..d8fa208f54 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -8,15 +8,20 @@ ENV DEBIAN_FRONTEND noninteractive # Install prerequisite packages for Arvados # * git, curl, rvm -# * Arvados source code in /usr/src/arvados-upstream, for preseeding gem installation +# * Arvados source code in /usr/src/arvados, for preseeding gem installation RUN apt-get update && \ - apt-get -q -y install -q -y openssh-server apt-utils git curl locales postgresql-server-dev-9.1 && \ + apt-get -q -y install -q -y openssh-server apt-utils git curl \ + libcurl3 libcurl3-gnutls libcurl4-openssl-dev locales \ + postgresql-server-dev-9.1 && \ /bin/mkdir -p /root/.ssh && \ /bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ /usr/sbin/locale-gen && \ - curl -L https://get.rvm.io | bash -s stable --ruby=2.1.0 && \ - git clone https://github.com/curoverse/arvados.git /usr/src/arvados-upstream + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 2.1.0 && \ + /bin/mkdir -p /usr/src/arvados + +ADD generated/arvados.tar.gz /usr/src/arvados/ # Set up RVM environment. These are just the env variables created by # /usr/local/rvm/scripts/rvm, which can't be run from a non-login shell. @@ -29,9 +34,9 @@ ENV PATH /usr/local/rvm/gems/ruby-2.1.0/bin:/usr/local/rvm/gems/ruby-2.1.0@globa # https://github.com/rubygems/rubygems.org/issues/613. RUN gem update --system && \ gem install bundler && \ - bundle install --gemfile=/usr/src/arvados-upstream/apps/workbench/Gemfile && \ - bundle install --gemfile=/usr/src/arvados-upstream/services/api/Gemfile && \ - bundle install --gemfile=/usr/src/arvados-upstream/doc/Gemfile + bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \ + bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \ + bundle install --gemfile=/usr/src/arvados/doc/Gemfile ADD generated/id_rsa.pub /root/.ssh/authorized_keys RUN chown root:root /root/.ssh/authorized_keys diff --git a/docker/build.rb b/docker/build.rb new file mode 100755 index 0000000000..ee2cc783ee --- /dev/null +++ b/docker/build.rb @@ -0,0 +1,185 @@ +#! /usr/bin/env ruby + +require 'tempfile' +require 'yaml' + +def sudo(*cmd) + # user can pass a single list in as an argument + # to allow usage like: sudo %w(apt-get install foo) + warn "You may need to enter your password here." + if cmd.length == 1 and cmd[0].class == Array + cmd = cmd[0] + end + system '/usr/bin/sudo', *cmd +end + +def is_valid_email? str + str.match /^\S+@\S+\.\S+$/ +end + +def generate_api_hostname + rand(2**256).to_s(36)[0...5] +end + +# ip_forwarding_enabled? +# Returns 'true' if IP forwarding is enabled in the kernel +# +def ip_forwarding_enabled? + %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n" +end + +def debootstrap_ok? + return system '/usr/sbin/debootstrap --version > /dev/null 2>&1' +end + +def docker_ok? + return system 'docker images > /dev/null 2>&1' +end + +# find_or_create_ssh_key arvados_name +# Return the SSH public key appropriate for this Arvados instance, +# generating one if necessary. +# +def find_or_create_ssh_key arvados_name + ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa" + unless File.exists? ssh_key_file + system 'ssh-keygen', + '-f', ssh_key_file, + '-C', "arvados@#{arvados_name}", + '-P', '' + end + + return "#{ssh_key_file}.pub" +end + +if not ip_forwarding_enabled? + warn "NOTE: IP forwarding must be enabled in the kernel." + warn "Turning IP forwarding on now." + sudo %w(/sbin/sysctl net.ipv4.ip_forward=1) +end + +# Check that: +# * Docker is installed and can be found in the user's path +# * Docker can be run as a non-root user +# - TODO: put the user is in the docker group if necessary +# - TODO: mount cgroup automatically +# - TODO: start the docker service if not started + +docker_path = %x(which docker).chomp +if docker_path.empty? + warn "Docker not found." + warn "" + warn "Please make sure that Docker has been installed and" + warn "can be found in your PATH." + warn "" + warn "Installation instructions for a variety of platforms can be found at" + warn "http://docs.docker.io/en/latest/installation/" + exit +elsif not docker_ok? + warn "WARNING: docker could not be run." + warn "Please make sure that:" + warn " * You have permission to read and write /var/run/docker.sock" + warn " * a 'cgroup' volume is mounted on your machine" + warn " * the docker daemon is running" + exit +end + +# Check that debootstrap is installed. +if not debootstrap_ok? + warn "Installing debootstrap." + sudo '/usr/bin/apt-get', 'install', 'debootstrap' +end + +# Generate a config.yml if it does not exist or is empty +if not File.size? 'config.yml' + print "Generating config.yml.\n" + print "Arvados needs to know the email address of the administrative user,\n" + print "so that when that user logs in they are automatically made an admin.\n" + print "This should be the email address you use to log in to Google.\n" + print "\n" + admin_email_address = "" + until is_valid_email? admin_email_address + print "Enter your Google ID email address here: " + admin_email_address = gets.strip + if not is_valid_email? admin_email_address + print "That doesn't look like a valid email address. Please try again.\n" + end + end + + File.open 'config.yml', 'w' do |config_out| + config = YAML.load_file 'config.yml.example' + config['API_AUTO_ADMIN_USER'] = admin_email_address + config['API_HOSTNAME'] = generate_api_hostname + config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME']) + config.each_key do |var| + if var.end_with?('_PW') or var.end_with?('_SECRET') + config[var] = rand(2**256).to_s(36) + end + config_out.write "#{var}: #{config[var]}\n" + end + end +end + +# If all prerequisites are met, go ahead and build. +if ip_forwarding_enabled? and + docker_ok? and + debootstrap_ok? and + File.exists? 'config.yml' + warn "Building Arvados." + system '/usr/bin/make', *ARGV +end + +# install_docker +# Determine which Docker package is suitable for this Linux distro +# and install, resolving any dependencies. +# NOTE: not in use yet. + +def install_docker + linux_distro = %x(lsb_release --id).split.last + linux_release = %x(lsb_release --release).split.last + linux_version = linux_distro + " " + linux_release + kernel_release = `uname -r` + + case linux_distro + when 'Ubuntu' + if not linux_release.match '^1[234]\.' + warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)." + warn "Your system is Ubuntu #{linux_release}." + exit + end + if linux_release.match '^12' and kernel_release.start_with? '3.2' + # Ubuntu Precise ships with a 3.2 kernel and must be upgraded. + warn "Your kernel #{kernel_release} must be upgraded to run Docker." + warn "To do this:" + warn " sudo apt-get update" + warn " sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring" + warn " sudo reboot" + exit + else + # install AUFS + sudo 'apt-get', 'update' + sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}" + end + + # add Docker repository + sudo %w(/usr/bin/apt-key adv + --keyserver keyserver.ubuntu.com + --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9) + source_file = Tempfile.new('arv') + source_file.write("deb http://get.docker.io/ubuntu docker main\n") + source_file.close + sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list' + sudo %w(/usr/bin/apt-get update) + sudo %w(/usr/bin/apt-get install lxc-docker) + + # Set up for non-root access + sudo %w(/usr/sbin/groupadd docker) + sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker' + sudo %w(/usr/sbin/service docker restart) + when 'Debian' + else + warn "Must be running a Debian or Ubuntu release in order to run Docker." + exit + end +end + diff --git a/docker/build.sh b/docker/build.sh old mode 100644 new mode 100755 index 6478f814a9..0e3d4cb6b5 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,42 +1,14 @@ #! /bin/bash -build_ok=true - -# Check that: -# * IP forwarding is enabled in the kernel. - -if [ "$(/sbin/sysctl --values net.ipv4.ip_forward)" != "1" ] -then - echo >&2 "WARNING: IP forwarding must be enabled in the kernel." - echo >&2 "Try: sudo sysctl net.ipv4.ip_forward=1" - build_ok=false -fi - -# * Docker can be found in the user's path -# * The user is in the docker group -# * cgroup is mounted -# * the docker daemon is running - -if ! docker images > /dev/null 2>&1 +# make sure Ruby 1.9.3 is installed before proceeding +if ! ruby -e 'exit RUBY_VERSION >= "1.9.3"' 2>/dev/null then - echo >&2 "WARNING: docker could not be run." - echo >&2 "Please make sure that:" - echo >&2 " * You have permission to read and write /var/run/docker.sock" - echo >&2 " * a 'cgroup' volume is mounted on your machine" - echo >&2 " * the docker daemon is running" - build_ok=false + echo "Installing Arvados requires at least Ruby 1.9.3." + echo "You may need to enter your password." + read -p "Press Ctrl-C to abort, or else press ENTER to install ruby1.9.3 and continue. " unused + + sudo apt-get update + sudo apt-get -y install ruby1.9.3 fi -# * config.yml exists -if [ '!' -f config.yml ] -then - echo >&2 "WARNING: no config.yml found in the current directory" - echo >&2 "Copy config.yml.example to config.yml and update it with settings for your site." - build_ok=false -fi - -# If ok to build, then go ahead and run make -if $build_ok -then - make $* -fi +./build.rb $* diff --git a/docker/config.rb b/docker/config.rb index 5e7242b4d5..ae69fe29ec 100755 --- a/docker/config.rb +++ b/docker/config.rb @@ -64,9 +64,8 @@ end # Copy the ssh public key file to base/generated (if a path is given) generated_dir = File.join('base/generated') Dir.mkdir(generated_dir) unless Dir.exists? generated_dir -if config.key?('PUBLIC_KEY_PATH') && - ! (config['PUBLIC_KEY_PATH'] == '') && - File.readable?(config['PUBLIC_KEY_PATH']) +if (config['PUBLIC_KEY_PATH'] != nil and + File.readable? config['PUBLIC_KEY_PATH']) FileUtils.cp(config['PUBLIC_KEY_PATH'], File.join(generated_dir, 'id_rsa.pub')) end diff --git a/docker/doc/Dockerfile b/docker/doc/Dockerfile index 3992881a85..6e4d1aa359 100644 --- a/docker/doc/Dockerfile +++ b/docker/doc/Dockerfile @@ -11,7 +11,8 @@ RUN /bin/mkdir -p /usr/src/arvados && \ ADD generated/doc.tar.gz /usr/src/arvados/ # Build static site -RUN /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \ +RUN bundle install --gemfile=/usr/src/arvados/doc/Gemfile && \ + /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \ cd /usr/src/arvados/doc && \ LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8" rake diff --git a/docker/install_sdk.sh b/docker/install_sdk.sh new file mode 100755 index 0000000000..1c07c9d821 --- /dev/null +++ b/docker/install_sdk.sh @@ -0,0 +1,13 @@ +#! /bin/sh + +# Install prerequisites. +sudo apt-get install curl libcurl3 libcurl3-gnutls libcurl4-openssl-dev python-pip + +# Install RVM. +curl -sSL https://get.rvm.io | bash -s stable +source ~/.rvm/scripts/rvm +rvm install 2.1.0 + +# Install arvados-cli. +gem install arvados-cli +sudo pip install --upgrade httplib2 diff --git a/docker/workbench/Dockerfile b/docker/workbench/Dockerfile index fea6947869..bd0bf4551e 100644 --- a/docker/workbench/Dockerfile +++ b/docker/workbench/Dockerfile @@ -6,13 +6,19 @@ MAINTAINER Ward Vandewege # Update Arvados source RUN /bin/mkdir -p /usr/src/arvados/apps ADD generated/workbench.tar.gz /usr/src/arvados/apps/ +ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb +ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb +ADD passenger.conf /etc/apache2/conf.d/passenger + -RUN touch /usr/src/arvados/apps/workbench/log/production.log && \ +RUN bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \ + touch /usr/src/arvados/apps/workbench/log/production.log && \ chmod 666 /usr/src/arvados/apps/workbench/log/production.log && \ touch /usr/src/arvados/apps/workbench/db/production.sqlite3 && \ bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \ cd /usr/src/arvados/apps/workbench && \ - rake assets:precompile + rake assets:precompile && \ + chown -R www-data:www-data /usr/src/arvados/apps/workbench # Configure Apache ADD generated/apache2_vhost /etc/apache2/sites-available/workbench @@ -21,11 +27,6 @@ RUN \ a2ensite workbench && \ a2enmod rewrite -# Set up the production environment -ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb -ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb -ADD passenger.conf /etc/apache2/conf.d/passenger - ADD apache2_foreground.sh /etc/apache2/foreground.sh # Start Apache diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index 4c8acbb59a..e6fa9fbb1f 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -71,9 +71,9 @@ use POSIX ':sys_wait_h'; use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); use Arvados; use Getopt::Long; -use Warehouse; -use Warehouse::Stream; -use IPC::System::Simple qw(capturex); +use IPC::Open2; +use IO::Select; +use File::Temp; $ENV{"TMPDIR"} ||= "/tmp"; unless (defined $ENV{"CRUNCH_TMP"}) { @@ -166,11 +166,8 @@ else } $job_id = $Job->{'uuid'}; -$metastream = Warehouse::Stream->new(whc => new Warehouse); -$metastream->clear; -$metastream->name('.'); -$metastream->write_start($job_id . '.log.txt'); - +my $keep_logfile = $job_id . '.log.txt'; +my $local_logfile = File::Temp->new(); $Job->{'runtime_constraints'} ||= {}; $Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0; @@ -733,7 +730,7 @@ if ($job_has_uuid) { if ($Job->{'output'}) { eval { - my $manifest_text = capturex("whget", $Job->{'output'}); + my $manifest_text = `arv keep get $Job->{'output'}`; $arv->{'collections'}->{'create'}->execute('collection' => { 'uuid' => $Job->{'output'}, 'manifest_text' => $manifest_text, @@ -1033,12 +1030,23 @@ sub process_stderr } split ("\n", $jobstep[$job]->{stderr}); } +sub fetch_block +{ + my $hash = shift; + my ($child_out, $child_in, $output_block); + + my $pid = open2($child_out, $child_in, 'arv', 'keep', 'get', $hash); + sysread($child_out, $output_block, 64 * 1024 * 1024); + waitpid($pid, 0); + return $output_block; +} sub collate_output { - my $whc = Warehouse->new; Log (undef, "collate"); - $whc->write_start (1); + + my ($child_out, $child_in); + my $pid = open2($child_out, $child_in, 'arv', 'keep', 'put', '--raw'); my $joboutput; for (@jobstep) { @@ -1049,26 +1057,31 @@ sub collate_output if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/) { $output_in_keep ||= $output =~ / [0-9a-f]{32}\S*\+K/; - $whc->write_data ($output); + print $child_in $output; } elsif (@jobstep == 1) { $joboutput = $output; - $whc->write_finish; + last; } - elsif (defined (my $outblock = $whc->fetch_block ($output))) + elsif (defined (my $outblock = fetch_block ($output))) { $output_in_keep ||= $outblock =~ / [0-9a-f]{32}\S*\+K/; - $whc->write_data ($outblock); + print $child_in $outblock; } else { - my $errstr = $whc->errstr; - $whc->write_data ("XXX fetch_block($output) failed: $errstr XXX\n"); + print $child_in "XXX fetch_block($output) failed XXX\n"; $main::success = 0; } } - $joboutput = $whc->write_finish if !defined $joboutput; + if (!defined $joboutput) { + my $s = IO::Select->new($child_out); + sysread($child_out, $joboutput, 64 * 1024 * 1024) if $s->can_read(5); + } + $child_in->close; + waitpid($pid, 0); + if ($joboutput) { Log (undef, "output $joboutput"); @@ -1143,8 +1156,9 @@ sub Log # ($jobstep_id, $logmessage) } print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message); - return if !$metastream; - $metastream->write_data ($datetime . " " . $message); + if ($metastream) { + print $metastream $datetime . " " . $message; + } } @@ -1173,16 +1187,14 @@ sub cleanup sub save_meta { my $justcheckpoint = shift; # false if this will be the last meta saved - my $m = $metastream; - $m = $m->copy if $justcheckpoint; - $m->write_finish; - my $whc = Warehouse->new; - my $loglocator = $whc->store_block ($m->as_string); - $arv->{'collections'}->{'create'}->execute('collection' => { - 'uuid' => $loglocator, - 'manifest_text' => $m->as_string, - }); - undef $metastream if !$justcheckpoint; # otherwise Log() will try to use it + return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm + + $local_logfile->flush; + my $cmd = "arv keep put --filename $keep_logfile ". $local_logfile->filename; + my $loglocator = `$cmd`; + die "system $cmd failed: $?" if $?; + + $local_logfile = undef; # the temp file is automatically deleted Log (undef, "log manifest is $loglocator"); $Job->{'log'} = $loglocator; $Job->update_attributes('log', $loglocator) if $job_has_uuid; @@ -1229,64 +1241,64 @@ sub thaw { croak ("Thaw not implemented"); - my $whc; - my $key = shift; - Log (undef, "thaw from $key"); - - @jobstep = (); - @jobstep_done = (); - @jobstep_todo = (); - @jobstep_tomerge = (); - $jobstep_tomerge_level = 0; - my $frozenjob = {}; - - my $stream = new Warehouse::Stream ( whc => $whc, - hash => [split (",", $key)] ); - $stream->rewind; - while (my $dataref = $stream->read_until (undef, "\n\n")) - { - if ($$dataref =~ /^job /) - { - foreach (split ("\n", $$dataref)) - { - my ($k, $v) = split ("=", $_, 2); - $frozenjob->{$k} = freezeunquote ($v); - } - next; - } - - if ($$dataref =~ /^merge (\d+) (.*)/) - { - $jobstep_tomerge_level = $1; - @jobstep_tomerge - = map { freezeunquote ($_) } split ("\n", freezeunquote($2)); - next; - } - - my $Jobstep = { }; - foreach (split ("\n", $$dataref)) - { - my ($k, $v) = split ("=", $_, 2); - $Jobstep->{$k} = freezeunquote ($v) if $k; - } - $Jobstep->{'failures'} = 0; - push @jobstep, $Jobstep; - - if ($Jobstep->{exitcode} eq "0") - { - push @jobstep_done, $#jobstep; - } - else - { - push @jobstep_todo, $#jobstep; - } - } - - foreach (qw (script script_version script_parameters)) - { - $Job->{$_} = $frozenjob->{$_}; - } - $Job->save if $job_has_uuid; + # my $whc; + # my $key = shift; + # Log (undef, "thaw from $key"); + + # @jobstep = (); + # @jobstep_done = (); + # @jobstep_todo = (); + # @jobstep_tomerge = (); + # $jobstep_tomerge_level = 0; + # my $frozenjob = {}; + + # my $stream = new Warehouse::Stream ( whc => $whc, + # hash => [split (",", $key)] ); + # $stream->rewind; + # while (my $dataref = $stream->read_until (undef, "\n\n")) + # { + # if ($$dataref =~ /^job /) + # { + # foreach (split ("\n", $$dataref)) + # { + # my ($k, $v) = split ("=", $_, 2); + # $frozenjob->{$k} = freezeunquote ($v); + # } + # next; + # } + + # if ($$dataref =~ /^merge (\d+) (.*)/) + # { + # $jobstep_tomerge_level = $1; + # @jobstep_tomerge + # = map { freezeunquote ($_) } split ("\n", freezeunquote($2)); + # next; + # } + + # my $Jobstep = { }; + # foreach (split ("\n", $$dataref)) + # { + # my ($k, $v) = split ("=", $_, 2); + # $Jobstep->{$k} = freezeunquote ($v) if $k; + # } + # $Jobstep->{'failures'} = 0; + # push @jobstep, $Jobstep; + + # if ($Jobstep->{exitcode} eq "0") + # { + # push @jobstep_done, $#jobstep; + # } + # else + # { + # push @jobstep_todo, $#jobstep; + # } + # } + + # foreach (qw (script script_version script_parameters)) + # { + # $Job->{$_} = $frozenjob->{$_}; + # } + # $Job->save if $job_has_uuid; } diff --git a/sdk/perl/Makefile.PL b/sdk/perl/Makefile.PL new file mode 100644 index 0000000000..21e31ad805 --- /dev/null +++ b/sdk/perl/Makefile.PL @@ -0,0 +1,10 @@ +#! /usr/bin/perl + +use strict; + +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => 'Arvados', + VERSION_FROM => 'lib/Arvados.pm' +);