X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f59d6d76acf9c6f5bb95c5902b2c9a1cca427e93..8e4ed0edeadc4dfcc85666ef5c36619815dca8e2:/sdk/cli/bin/crunch-job diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index 7aa02ac699..a6ccebcd91 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -86,6 +86,7 @@ use POSIX ':sys_wait_h'; use POSIX qw(strftime); use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); use Arvados; +use Cwd qw(realpath); use Data::Dumper; use Digest::MD5 qw(md5_hex); use Getopt::Long; @@ -95,6 +96,7 @@ use File::Temp; use Fcntl ':flock'; use File::Path qw( make_path remove_tree ); +use constant TASK_TEMPFAIL => 111; use constant EX_TEMPFAIL => 75; $ENV{"TMPDIR"} ||= "/tmp"; @@ -197,6 +199,16 @@ $Job->{'runtime_constraints'} ||= {}; $Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0; my $max_ncpus = $Job->{'runtime_constraints'}->{'max_tasks_per_node'}; +my $gem_versions = `gem list --quiet arvados-cli 2>/dev/null`; +if ($? == 0) { + $gem_versions =~ s/^arvados-cli \(/ with arvados-cli Gem version(s) /; + chomp($gem_versions); + chop($gem_versions); # Closing parentheses +} else { + $gem_versions = ""; +} +Log(undef, + "running from " . ((-e $0) ? realpath($0) : "stdin") . $gem_versions); Log (undef, "check slurm allocation"); my @slot; @@ -334,13 +346,9 @@ if (!$have_slurm) must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here."); } - -my $build_script; -do { - local $/ = undef; - $build_script = ; -}; +my $build_script = handle_readall(\*DATA); my $nodelist = join(",", @node); +my $git_tar_count = 0; if (!defined $no_clear_tmp) { # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src* @@ -349,8 +357,12 @@ if (!defined $no_clear_tmp) { my $cleanpid = fork(); if ($cleanpid == 0) { + # Find FUSE mounts that look like Keep mounts (the mount path has the + # word "keep") and unmount them. Then clean up work directories. + # TODO: When #5036 is done and widely deployed, we can get rid of the + # regular expression and just unmount everything with type fuse.keep. srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}], - ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep $CRUNCH_TMP/task/*.keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src*']); + ['bash', '-ec', 'mount -t fuse,fuse.keep | awk \'($3 ~ /\ykeep\y/){print $3}\' | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid']); exit (1); } while (1) @@ -362,8 +374,57 @@ if (!defined $no_clear_tmp) { Log (undef, "Cleanup command exited ".exit_status_s($?)); } +# If this job requires a Docker image, install that. +my $docker_bin = "/usr/bin/docker.io"; +my ($docker_locator, $docker_stream, $docker_hash, $docker_limitmem); +if ($docker_locator = $Job->{docker_image_locator}) { + ($docker_stream, $docker_hash) = find_docker_image($docker_locator); + if (!$docker_hash) + { + croak("No Docker image hash found from locator $docker_locator"); + } + $docker_stream =~ s/^\.//; + my $docker_install_script = qq{ +if ! $docker_bin images -q --no-trunc --all | grep -qxF \Q$docker_hash\E; then + arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load +fi +}; + my $docker_pid = fork(); + if ($docker_pid == 0) + { + srun (["srun", "--nodelist=" . join(',', @node)], + ["/bin/sh", "-ec", $docker_install_script]); + exit ($?); + } + while (1) + { + last if $docker_pid == waitpid (-1, WNOHANG); + freeze_if_want_freeze ($docker_pid); + select (undef, undef, undef, 0.1); + } + if ($? != 0) + { + croak("Installing Docker image from $docker_locator exited " + .exit_status_s($?)); + } + + # Determine whether this version of Docker supports memory+swap limits. + srun(["srun", "--nodelist=" . $node[0]], + ["/bin/sh", "-ec", "$docker_bin run --help | grep -qe --memory-swap="], + {fork => 1}); + $docker_limitmem = ($? == 0); + + if ($Job->{arvados_sdk_version}) { + # The job also specifies an Arvados SDK version. Add the SDKs to the + # tar file for the build script to install. + Log(undef, sprintf("Packing Arvados SDK version %s for installation", + $Job->{arvados_sdk_version})); + add_git_archive("git", "--git-dir=$git_dir", "archive", + "--prefix=.arvados.sdk/", + $Job->{arvados_sdk_version}, "sdk"); + } +} -my $git_archive; if (!defined $git_dir && $Job->{'script_version'} =~ m{^/}) { # If script_version looks like an absolute path, *and* the --git-dir # argument was not given -- which implies we were not invoked by @@ -517,12 +578,10 @@ else { } $ENV{"CRUNCH_SRC_COMMIT"} = $commit; - $git_archive = `$gitcmd archive ''\Q$commit\E`; - if ($?) { - croak("Error: $gitcmd archive exited ".exit_status_s($?)); - } + add_git_archive("$gitcmd archive ''\Q$commit\E"); } +my $git_archive = combined_git_archive(); if (!defined $git_archive) { Log(undef, "Skip install phase (no git archive)"); if ($have_slurm) { @@ -552,48 +611,10 @@ else { } my $install_exited = $?; Log (undef, "Install script exited ".exit_status_s($install_exited)); - exit (1) if $install_exited != 0; -} - -if (!$have_slurm) -{ - # Grab our lock again (we might have deleted and re-created CRUNCH_TMP above) - must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here."); -} - -# If this job requires a Docker image, install that. -my $docker_bin = "/usr/bin/docker.io"; -my ($docker_locator, $docker_stream, $docker_hash); -if ($docker_locator = $Job->{docker_image_locator}) { - ($docker_stream, $docker_hash) = find_docker_image($docker_locator); - if (!$docker_hash) - { - croak("No Docker image hash found from locator $docker_locator"); - } - $docker_stream =~ s/^\.//; - my $docker_install_script = qq{ -if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then - arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load -fi -}; - my $docker_pid = fork(); - if ($docker_pid == 0) - { - srun (["srun", "--nodelist=" . join(',', @node)], - ["/bin/sh", "-ec", $docker_install_script]); - exit ($?); - } - while (1) - { - last if $docker_pid == waitpid (-1, WNOHANG); - freeze_if_want_freeze ($docker_pid); - select (undef, undef, undef, 0.1); - } - if ($? != 0) - { - croak("Installing Docker image from $docker_locator exited " - .exit_status_s($?)); + foreach my $tar_filename (map { tar_filename_n($_); } (1..$git_tar_count)) { + unlink($tar_filename); } + exit (1) if $install_exited != 0; } foreach (qw (script script_version script_parameters runtime_constraints)) @@ -638,6 +659,10 @@ update_progress_stats(); THISROUND: +my $tasks_this_level = 0; +foreach my $id (@jobstep_todo) { + $tasks_this_level++ if ($jobstep[$id]->{level} == $level); +} for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) { my $id = $jobstep_todo[$todo_ptr]; @@ -656,6 +681,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) my $childslotname = join (".", $slot[$childslot]->{node}->{name}, $slot[$childslot]->{cpu}); + my $childpid = fork(); if ($childpid == 0) { @@ -694,6 +720,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) $ENV{"GZIP"} = "-n"; + my $max_node_concurrent_tasks = $ENV{CRUNCH_NODE_SLOTS}; + if ($tasks_this_level < $max_node_concurrent_tasks) { + $max_node_concurrent_tasks = $tasks_this_level; + } + my @srunargs = ( "srun", "--nodelist=".$childnode->{name}, @@ -703,13 +734,26 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) my $command = "if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; " ."mkdir -p $ENV{CRUNCH_TMP} $ENV{JOB_WORK} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} " - ."&& cd $ENV{CRUNCH_TMP} "; + ."&& cd $ENV{CRUNCH_TMP} " + # These environment variables get used explicitly later in + # $command. No tool is expected to read these values directly. + .q{&& MEM=$(awk '($1 == "MemTotal:"){print $2}' @freeslot && $todo_ptr+1 > $#jobstep_todo)) { - last THISROUND if $main::please_freeze; + last THISROUND if $main::please_freeze || defined($main::success); if ($main::please_info) { $main::please_info = 0; @@ -927,7 +971,7 @@ if (!$collated_output) { Log (undef, "Failed to write output collection"); } else { - Log(undef, "output hash " . $collated_output); + Log(undef, "job output $collated_output"); $Job->update_attributes('output' => $collated_output); } @@ -996,7 +1040,7 @@ sub reapchildren { my $temporary_fail; $temporary_fail ||= $Jobstep->{node_fail}; - $temporary_fail ||= ($exitvalue == 111); + $temporary_fail ||= ($exitvalue == TASK_TEMPFAIL); ++$thisround_failed; ++$thisround_failed_multiple if $Jobstep->{'failures'} >= 1; @@ -1023,7 +1067,6 @@ sub reapchildren if (!$temporary_fail || $Jobstep->{'failures'} >= 3) { # Give up on this task, and the whole job $main::success = 0; - $main::please_freeze = 1; } # Put this task back on the todo queue push @jobstep_todo, $jobstepid; @@ -1042,7 +1085,9 @@ sub reapchildren $Jobstep->{'arvados_task'}->{finished_at} = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($Jobstep->{finishtime}); $Jobstep->{'arvados_task'}->save; process_stderr ($jobstepid, $task_success); - Log ($jobstepid, "output " . $Jobstep->{'arvados_task'}->{output}); + Log ($jobstepid, sprintf("task output (%d bytes): %s", + length($Jobstep->{'arvados_task'}->{output}), + $Jobstep->{'arvados_task'}->{output})); close $reader{$jobstepid}; delete $reader{$jobstepid}; @@ -1216,7 +1261,7 @@ sub preprocess_stderr # whoa. $main::please_freeze = 1; } - elsif ($line =~ /srun: error: (Node failure on|Unable to create job step) /) { + elsif ($line =~ /(srun: error: (Node failure on|Unable to create job step|.*: Communication connection failure))|arvados.errors.Keep/) { $jobstep[$job]->{node_fail} = 1; ban_node_by_slot($jobstep[$job]->{slotindex}); } @@ -1238,16 +1283,19 @@ sub process_stderr sub fetch_block { my $hash = shift; - my ($keep, $child_out, $output_block); - - my $cmd = "arv-get \Q$hash\E"; - open($keep, '-|', $cmd) or die "fetch_block: $cmd: $!"; - $output_block = ''; + my $keep; + if (!open($keep, "-|", "arv-get", "--retries", retry_count(), $hash)) { + Log(undef, "fetch_block run error from arv-get $hash: $!"); + return undef; + } + my $output_block = ""; while (1) { my $buf; my $bytes = sysread($keep, $buf, 1024 * 1024); if (!defined $bytes) { - die "reading from arv-get: $!"; + Log(undef, "fetch_block read error from arv-get: $!"); + $output_block = undef; + last; } elsif ($bytes == 0) { # sysread returns 0 at the end of the pipe. last; @@ -1257,60 +1305,80 @@ sub fetch_block } } close $keep; + if ($?) { + Log(undef, "fetch_block arv-get exited " . exit_status_s($?)); + $output_block = undef; + } return $output_block; } -# create_output_collections generates a new collection containing the -# output of each successfully completed task, and returns the -# portable_data_hash for the new collection. -# +# Create a collection by concatenating the output of all tasks (each +# task's output is either a manifest fragment, a locator for a +# manifest fragment stored in Keep, or nothing at all). Return the +# portable_data_hash of the new collection. sub create_output_collection { Log (undef, "collate"); my ($child_out, $child_in); - my $pid = open2($child_out, $child_in, 'python', '-c', - 'import arvados; ' . - 'import sys; ' . - 'print arvados.api()' . - '.collections()' . - '.create(body={"manifest_text":sys.stdin.read()})' . - '.execute()["portable_data_hash"]' - ); - + my $pid = open2($child_out, $child_in, 'python', '-c', q{ +import arvados +import sys +print (arvados.api("v1").collections(). + create(body={"manifest_text": sys.stdin.read()}). + execute(num_retries=int(sys.argv[1]))["portable_data_hash"]) +}, retry_count()); + + my $task_idx = -1; + my $manifest_size = 0; for (@jobstep) { - next if (!exists $_->{'arvados_task'}->{'output'} || - !$_->{'arvados_task'}->{'success'}); + ++$task_idx; my $output = $_->{'arvados_task'}->{output}; - if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/) - { - print $child_in $output; - } - elsif (defined (my $outblock = fetch_block ($output))) - { - print $child_in $outblock; + next if (!defined($output)); + my $next_write; + if ($output =~ /^[0-9a-f]{32}(\+\S+)*$/) { + $next_write = fetch_block($output); + } else { + $next_write = $output; } - else - { - Log (undef, "XXX fetch_block($output) failed XXX"); + if (defined($next_write)) { + if (!defined(syswrite($child_in, $next_write))) { + # There's been an error writing. Stop the loop. + # We'll log details about the exit code later. + last; + } else { + $manifest_size += length($next_write); + } + } else { + my $uuid = $_->{'arvados_task'}->{'uuid'}; + Log (undef, "Error retrieving '$output' output by task $task_idx ($uuid)"); $main::success = 0; } } - $child_in->close; + close($child_in); + Log(undef, "collated output manifest text to send to API server is $manifest_size bytes with access tokens"); my $joboutput; my $s = IO::Select->new($child_out); if ($s->can_read(120)) { - sysread($child_out, $joboutput, 64 * 1024 * 1024); - chomp($joboutput); - # TODO: Ensure exit status == 0. + sysread($child_out, $joboutput, 1024 * 1024); + waitpid($pid, 0); + if ($?) { + Log(undef, "output collection creation exited " . exit_status_s($?)); + $joboutput = undef; + } else { + chomp($joboutput); + } } else { Log (undef, "timed out while creating output collection"); + foreach my $signal (2, 2, 2, 15, 15, 9) { + kill($signal, $pid); + last if waitpid($pid, WNOHANG) == -1; + sleep(1); + } } - # TODO: kill $pid instead of waiting, now that we've decided to - # ignore further output. - waitpid($pid, 0); + close($child_out); return $joboutput; } @@ -1384,8 +1452,11 @@ sub log_writer_start($) { my $logfilename = shift; $log_pipe_pid = open2($log_pipe_out, $log_pipe_in, - 'arv-put', '--portable-data-hash', + 'arv-put', + '--portable-data-hash', + '--project-uuid', $Job->{owner_uuid}, '--retries', '3', + '--name', $logfilename, '--filename', $logfilename, '-'); } @@ -1715,17 +1786,89 @@ sub exit_status_s { return $s; } +sub handle_readall { + # Pass in a glob reference to a file handle. + # Read all its contents and return them as a string. + my $fh_glob_ref = shift; + local $/ = undef; + return <$fh_glob_ref>; +} + +sub tar_filename_n { + my $n = shift; + return sprintf("%s/git.%s.%d.tar", $ENV{CRUNCH_TMP}, $job_id, $n); +} + +sub add_git_archive { + # Pass in a git archive command as a string or list, a la system(). + # This method will save its output to be included in the archive sent to the + # build script. + my $git_input; + $git_tar_count++; + if (!open(GIT_ARCHIVE, ">", tar_filename_n($git_tar_count))) { + croak("Failed to save git archive: $!"); + } + my $git_pid = open2(">&GIT_ARCHIVE", $git_input, @_); + close($git_input); + waitpid($git_pid, 0); + close(GIT_ARCHIVE); + if ($?) { + croak("Failed to save git archive: git exited " . exit_status_s($?)); + } +} + +sub combined_git_archive { + # Combine all saved tar archives into a single archive, then return its + # contents in a string. Return undef if no archives have been saved. + if ($git_tar_count < 1) { + return undef; + } + my $base_tar_name = tar_filename_n(1); + foreach my $tar_to_append (map { tar_filename_n($_); } (2..$git_tar_count)) { + my $tar_exit = system("tar", "-Af", $base_tar_name, $tar_to_append); + if ($tar_exit != 0) { + croak("Error preparing build archive: tar -A exited " . + exit_status_s($tar_exit)); + } + } + if (!open(GIT_TAR, "<", $base_tar_name)) { + croak("Could not open build archive: $!"); + } + my $tar_contents = handle_readall(\*GIT_TAR); + close(GIT_TAR); + return $tar_contents; +} + __DATA__ #!/usr/bin/perl - -# checkout-and-build +# +# This is crunch-job's internal dispatch script. crunch-job running on the API +# server invokes this script on individual compute nodes, or localhost if we're +# running a job locally. It gets called in two modes: +# +# * No arguments: Installation mode. Read a tar archive from the DATA +# file handle; it includes the Crunch script's source code, and +# maybe SDKs as well. Those should be installed in the proper +# locations. This runs outside of any Docker container, so don't try to +# introspect Crunch's runtime environment. +# +# * With arguments: Crunch script run mode. This script should set up the +# environment, then run the command specified in the arguments. This runs +# inside any Docker container. use Fcntl ':flock'; use File::Path qw( make_path remove_tree ); +use POSIX qw(getcwd); + +use constant TASK_TEMPFAIL => 111; + +# Map SDK subdirectories to the path environments they belong to. +my %SDK_ENVVARS = ("perl/lib" => "PERLLIB", "ruby/lib" => "RUBYLIB"); my $destdir = $ENV{"CRUNCH_SRC"}; my $commit = $ENV{"CRUNCH_SRC_COMMIT"}; my $repo = $ENV{"CRUNCH_SRC_URL"}; +my $install_dir = $ENV{"CRUNCH_INSTALL"} || (getcwd() . "/opt"); my $job_work = $ENV{"JOB_WORK"}; my $task_work = $ENV{"TASK_WORK"}; @@ -1740,50 +1883,134 @@ if ($task_work) { remove_tree($task_work, {keep_root => 1}); } -my @git_archive_data = ; -if (!@git_archive_data) { - # Nothing to extract -> nothing to install. - run_argv_and_exit(); +open(STDOUT_ORIG, ">&", STDOUT); +open(STDERR_ORIG, ">&", STDERR); +open(STDOUT, ">>", "$destdir.log"); +open(STDERR, ">&", STDOUT); + +### Crunch script run mode +if (@ARGV) { + # We want to do routine logging during task 0 only. This gives the user + # the information they need, but avoids repeating the information for every + # task. + my $Log; + if ($ENV{TASK_SEQUENCE} eq "0") { + $Log = sub { + my $msg = shift; + printf STDERR_ORIG "[Crunch] $msg\n", @_; + }; + } else { + $Log = sub { }; + } + + my $python_src = "$install_dir/python"; + my $venv_dir = "$job_work/.arvados.venv"; + my $venv_built = -e "$venv_dir/bin/activate"; + if ((!$venv_built) and (-d $python_src) and can_run("virtualenv")) { + shell_or_die(undef, "virtualenv", "--quiet", "--system-site-packages", + "--python=python2.7", $venv_dir); + shell_or_die(TASK_TEMPFAIL, "$venv_dir/bin/pip", "--quiet", "install", "-I", $python_src); + $venv_built = 1; + $Log->("Built Python SDK virtualenv"); + } + + my $pip_bin = "pip"; + if ($venv_built) { + $Log->("Running in Python SDK virtualenv"); + $pip_bin = "$venv_dir/bin/pip"; + my $orig_argv = join(" ", map { quotemeta($_); } @ARGV); + @ARGV = ("/bin/sh", "-ec", + ". \Q$venv_dir/bin/activate\E; exec $orig_argv"); + } elsif (-d $python_src) { + $Log->("Warning: virtualenv not found inside Docker container default " . + "\$PATH. Can't install Python SDK."); + } + + my $pkgs = `(\Q$pip_bin\E freeze 2>/dev/null | grep arvados) || dpkg-query --show '*arvados*'`; + if ($pkgs) { + $Log->("Using Arvados SDK:"); + foreach my $line (split /\n/, $pkgs) { + $Log->($line); + } + } else { + $Log->("Arvados SDK packages not found"); + } + + while (my ($sdk_dir, $sdk_envkey) = each(%SDK_ENVVARS)) { + my $sdk_path = "$install_dir/$sdk_dir"; + if (-d $sdk_path) { + if ($ENV{$sdk_envkey}) { + $ENV{$sdk_envkey} = "$sdk_path:" . $ENV{$sdk_envkey}; + } else { + $ENV{$sdk_envkey} = $sdk_path; + } + $Log->("Arvados SDK added to %s", $sdk_envkey); + } + } + + close(STDOUT); + close(STDERR); + open(STDOUT, ">&", STDOUT_ORIG); + open(STDERR, ">&", STDERR_ORIG); + exec(@ARGV); + die "Cannot exec `@ARGV`: $!"; } +### Installation mode open L, ">", "$destdir.lock" or die "$destdir.lock: $!"; flock L, LOCK_EX; if (readlink ("$destdir.commit") eq $commit && -d $destdir) { # This version already installed -> nothing to do. - run_argv_and_exit(); + exit(0); } unlink "$destdir.commit"; -open STDERR_ORIG, ">&STDERR"; -open STDOUT, ">", "$destdir.log"; -open STDERR, ">&STDOUT"; - mkdir $destdir; -open TARX, "|-", "tar", "-C", $destdir, "-xf", "-"; -print TARX @git_archive_data; + +if (!open(TARX, "|-", "tar", "-xC", $destdir)) { + die "Error launching 'tar -xC $destdir': $!"; +} +# If we send too much data to tar in one write (> 4-5 MiB), it stops, and we +# get SIGPIPE. We must feed it data incrementally. +my $tar_input; +while (read(DATA, $tar_input, 65536)) { + print TARX $tar_input; +} if(!close(TARX)) { - die "'tar -C $destdir -xf -' exited $?: $!"; + die "'tar -xC $destdir' exited $?: $!"; } -my $pwd; -chomp ($pwd = `pwd`); -my $install_dir = $ENV{"CRUNCH_INSTALL"} || "$pwd/opt"; mkdir $install_dir; -for my $src_path ("$destdir/arvados/sdk/python") { - if (-d $src_path) { - shell_or_die ("virtualenv", $install_dir); - shell_or_die ("cd $src_path && ./build.sh && $install_dir/bin/python setup.py install"); +my $sdk_root = "$destdir/.arvados.sdk/sdk"; +if (-d $sdk_root) { + foreach my $sdk_lang (("python", + map { (split /\//, $_, 2)[0]; } keys(%SDK_ENVVARS))) { + if (-d "$sdk_root/$sdk_lang") { + if (!rename("$sdk_root/$sdk_lang", "$install_dir/$sdk_lang")) { + die "Failed to install $sdk_lang SDK: $!"; + } + } } } +my $python_dir = "$install_dir/python"; +if ((-d $python_dir) and can_run("python2.7") and + (system("python2.7", "$python_dir/setup.py", "--quiet", "egg_info") != 0)) { + # egg_info failed, probably when it asked git for a build tag. + # Specify no build tag. + open(my $pysdk_cfg, ">>", "$python_dir/setup.cfg"); + print $pysdk_cfg "\n[egg_info]\ntag_build =\n"; + close($pysdk_cfg); +} + if (-e "$destdir/crunch_scripts/install") { - shell_or_die ("$destdir/crunch_scripts/install", $install_dir); + shell_or_die (undef, "$destdir/crunch_scripts/install", $install_dir); } elsif (!-e "./install.sh" && -e "./tests/autotests.sh") { # Old version - shell_or_die ("./tests/autotests.sh", $install_dir); + shell_or_die (undef, "./tests/autotests.sh", $install_dir); } elsif (-e "./install.sh") { - shell_or_die ("./install.sh", $install_dir); + shell_or_die (undef, "./install.sh", $install_dir); } if ($commit) { @@ -1794,29 +2021,34 @@ if ($commit) { close L; -run_argv_and_exit(); - -sub run_argv_and_exit -{ - if (@ARGV) { - exec(@ARGV); - die "Cannot exec `@ARGV`: $!"; - } else { - exit 0; - } +sub can_run { + my $command_name = shift; + open(my $which, "-|", "which", $command_name); + while (<$which>) { } + close($which); + return ($? == 0); } sub shell_or_die { + my $exitcode = shift; + if ($ENV{"DEBUG"}) { print STDERR "@_\n"; } if (system (@_) != 0) { my $err = $!; - my $exitstatus = sprintf("exit %d signal %d", $? >> 8, $? & 0x7f); + my $code = $?; + my $exitstatus = sprintf("exit %d signal %d", $code >> 8, $code & 0x7f); open STDERR, ">&STDERR_ORIG"; system ("cat $destdir.log >&2"); - die "@_ failed ($err): $exitstatus"; + warn "@_ failed ($err): $exitstatus"; + if (defined($exitcode)) { + exit $exitcode; + } + else { + exit (($code >> 8) || 1); + } } }