X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/283dbf37a1b7d32332e295070de48b5e6e459248..2c6c1cb11153849e87496bb8d1f5ff24f439a6a4:/sdk/cli/bin/crunch-job diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index e185d66d69..da005a3884 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -95,6 +95,15 @@ $ENV{"CRUNCH_INSTALL"} = "$ENV{CRUNCH_TMP}/opt"; $ENV{"CRUNCH_WORK"} = $ENV{"JOB_WORK"}; # deprecated mkdir ($ENV{"JOB_WORK"}); +my $arv_cli; + +if (defined $ENV{"ARV_CLI"}) { + $arv_cli = $ENV{"ARV_CLI"}; +} +else { + $arv_cli = 'arv'; +} + my $force_unlock; my $git_dir; my $jobspec; @@ -130,7 +139,7 @@ $SIG{'USR2'} = sub my $arv = Arvados->new('apiVersion' => 'v1'); -my $metastream; +my $local_logfile; my $User = $arv->{'users'}->{'current'}->execute; @@ -176,7 +185,7 @@ else $job_id = $Job->{'uuid'}; my $keep_logfile = $job_id . '.log.txt'; -my $local_logfile = File::Temp->new(); +$local_logfile = File::Temp->new(); $Job->{'runtime_constraints'} ||= {}; $Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0; @@ -195,7 +204,7 @@ if (!$have_slurm) } if (exists $ENV{SLURM_NODELIST}) { - push @sinfo, `sinfo -h --format='%c %N' --nodes='$ENV{SLURM_NODELIST}'`; + push @sinfo, `sinfo -h --format='%c %N' --nodes=\Q$ENV{SLURM_NODELIST}\E`; } foreach (@sinfo) { @@ -400,24 +409,33 @@ else my $commit; my $git_archive; my $treeish = $Job->{'script_version'}; - my $repo = $git_dir || $ENV{'CRUNCH_DEFAULT_GIT_DIR'}; - # Todo: let script_version specify repository instead of expecting - # parent process to figure it out. - $ENV{"CRUNCH_SRC_URL"} = $repo; - # Create/update our clone of the remote git repo + # If we're running under crunch-dispatch, it will have pulled the + # appropriate source tree into its own repository, and given us that + # repo's path as $git_dir. If we're running a "local" job, and a + # script_version was specified, it's up to the user to provide the + # full path to a local repository in Job->{repository}. + # + # TODO: Accept URLs too, not just local paths. Use git-ls-remote and + # git-archive --remote where appropriate. + # + # TODO: Accept a locally-hosted Arvados repository by name or + # UUID. Use arvados.v1.repositories.list or .get to figure out the + # appropriate fetch-url. + my $repo = $git_dir || $ENV{'CRUNCH_DEFAULT_GIT_DIR'} || $Job->{'repository'}; + + $ENV{"CRUNCH_SRC_URL"} = $repo; - if (!-d $ENV{"CRUNCH_SRC"}) { - system(qw(git clone), $repo, $ENV{"CRUNCH_SRC"}) == 0 - or croak ("git clone $repo failed: exit ".($?>>8)); - system("cd $ENV{CRUNCH_SRC} && git config clean.requireForce false"); + if (-d "$repo/.git") { + # We were given a working directory, but we are only interested in + # the index. + $repo = "$repo/.git"; } - `cd $ENV{CRUNCH_SRC} && git remote set-url origin \"\$CRUNCH_SRC_URL\" && git fetch -q --tags origin`; # If this looks like a subversion r#, look for it in git-svn commit messages if ($treeish =~ m{^\d{1,4}$}) { - my $gitlog = `cd $ENV{CRUNCH_SRC} && git log --pretty="format:%H" --grep="git-svn-id:.*\@$treeish " origin/master`; + my $gitlog = `git --git-dir=\Q$repo\E log --pretty="format:%H" --grep="git-svn-id:.*\@"\Q$treeish\E" " master`; chomp $gitlog; if ($gitlog =~ /^[a-f0-9]{40}$/) { $commit = $gitlog; @@ -428,15 +446,7 @@ else # If that didn't work, try asking git to look it up as a tree-ish. if (!defined $commit) { - - my $cooked_treeish = $treeish; - if ($treeish !~ m{^[0-9a-f]{5,}$}) { - # Looks like a git branch name -- make sure git knows it's - # relative to the remote repo - $cooked_treeish = "origin/$treeish"; - } - - my $found = `cd $ENV{CRUNCH_SRC} && git rev-list -1 $cooked_treeish`; + my $found = `git --git-dir=\Q$repo\E rev-list -1 ''\Q$treeish\E`; chomp $found; if ($found =~ /^[0-9a-f]{40}$/s) { $commit = $found; @@ -461,7 +471,7 @@ else $ENV{"CRUNCH_SRC_COMMIT"} = $commit; @execargs = ("sh", "-c", "mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -"); - $git_archive = `cd $ENV{CRUNCH_SRC} && git archive $commit`; + $git_archive = `git --git-dir=\Q$repo\E archive ''\Q$commit\E`; } else { croak ("could not figure out commit id for $treeish"); @@ -488,7 +498,30 @@ if (!$have_slurm) must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here."); } - +# If this job requires a Docker image, install that. +my $docker_bin = "/usr/bin/docker.io"; +my $docker_image = $Job->{runtime_constraints}->{docker_image} || ""; +if ($docker_image) { + my $docker_pid = fork(); + if ($docker_pid == 0) + { + srun (["srun", "--nodelist=" . join(' ', @node)], + [$docker_bin, 'pull', $docker_image]); + exit ($?); + } + while (1) + { + last if $docker_pid == waitpid (-1, WNOHANG); + freeze_if_want_freeze ($docker_pid); + select (undef, undef, undef, 0.1); + } + # If the Docker image was specified as a hash, pull will fail. + # Ignore that error. We'll see what happens when we try to run later. + if (($? != 0) && ($docker_image !~ /^[0-9a-fA-F]{5,64}$/)) + { + croak("Installing Docker image $docker_image returned exit code $?"); + } +} foreach (qw (script script_version script_parameters runtime_constraints)) { @@ -593,7 +626,6 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'}, "--job-name=$job_id.$id.$$", ); - my @execargs = qw(sh); my $build_script_to_send = ""; my $command = "if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; " @@ -605,8 +637,32 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) $command .= "&& perl -"; } - $command .= - "&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"}; + $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec "; + if ($docker_image) + { + $command .= "crunchstat -cgroup-parent=/sys/fs/cgroup/lxc -cgroup-cid=$ENV{TASK_WORK}/docker.cid "; + $command .= "$docker_bin run -i -a stdin -a stdout -a stderr -cidfile=$ENV{TASK_WORK}/docker.cid "; + # Dynamically configure the container to use the host system as its + # DNS server. Get the host's global addresses from the ip command, + # and turn them into docker --dns options using gawk. + $command .= + q{$(ip -o address show scope global | + gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') }; + foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT)) + { + $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E "; + } + while (my ($env_key, $env_val) = each %ENV) + { + if ($env_key =~ /^(JOB|TASK)_/) { + $command .= "-e \Q$env_key=$env_val\E "; + } + } + $command .= "\Q$docker_image\E "; + } else { + $command .= "crunchstat -cgroup-path=/sys/fs/cgroup " + } + $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"}; my @execargs = ('bash', '-c', $command); srun (\@srunargs, \@execargs, undef, $build_script_to_send); exit (111); @@ -758,7 +814,7 @@ if ($job_has_uuid) { if ($Job->{'output'}) { eval { - my $manifest_text = `arv keep get \Q$Job->{'output'}\E`; + my $manifest_text = `arv keep get ''\Q$Job->{'output'}\E`; $arv->{'collections'}->{'create'}->execute('collection' => { 'uuid' => $Job->{'output'}, 'manifest_text' => $manifest_text, @@ -895,13 +951,19 @@ sub reapchildren delete $proc{$pid}; # Load new tasks - my $newtask_list = $arv->{'job_tasks'}->{'list'}->execute( - 'where' => { - 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid} - }, - 'order' => 'qsequence' - ); - foreach my $arvados_task (@{$newtask_list->{'items'}}) { + my $newtask_list = []; + my $newtask_results; + do { + $newtask_results = $arv->{'job_tasks'}->{'list'}->execute( + 'where' => { + 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid} + }, + 'order' => 'qsequence', + 'offset' => scalar(@$newtask_list), + ); + push(@$newtask_list, @{$newtask_results->{items}}); + } while (@{$newtask_results->{items}}); + foreach my $arvados_task (@$newtask_list) { my $jobstep = { 'level' => $arvados_task->{'sequence'}, 'failures' => 0, @@ -1073,7 +1135,7 @@ sub fetch_block my $hash = shift; my ($keep, $child_out, $output_block); - my $cmd = "arv keep get \Q$hash\E"; + my $cmd = "$arv_cli keep get \Q$hash\E"; open($keep, '-|', $cmd) or die "fetch_block: $cmd: $!"; sysread($keep, $output_block, 64 * 1024 * 1024); close $keep; @@ -1085,7 +1147,7 @@ sub collate_output Log (undef, "collate"); my ($child_out, $child_in); - my $pid = open2($child_out, $child_in, 'arv', 'keep', 'put', '--raw'); + my $pid = open2($child_out, $child_in, $arv_cli, 'keep', 'put', '--raw'); my $joboutput; for (@jobstep) { @@ -1194,15 +1256,15 @@ sub Log # ($jobstep_id, $logmessage) $message =~ s{([^ -\176])}{"\\" . sprintf ("%03o", ord($1))}ge; $message .= "\n"; my $datetime; - if ($metastream || -t STDERR) { + if ($local_logfile || -t STDERR) { my @gmtime = gmtime; $datetime = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d", $gmtime[5]+1900, $gmtime[4]+1, @gmtime[3,2,1,0]); } print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message); - if ($metastream) { - print $metastream $datetime . " " . $message; + if ($local_logfile) { + print $local_logfile $datetime . " " . $message; } } @@ -1215,7 +1277,7 @@ sub croak freeze() if @jobstep_todo; collate_output() if @jobstep_todo; cleanup(); - save_meta() if $metastream; + save_meta() if $local_logfile; die; } @@ -1235,10 +1297,11 @@ sub save_meta return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm $local_logfile->flush; - my $cmd = "arv keep put --filename \Q$keep_logfile\E " + my $cmd = "$arv_cli keep put --filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename); my $loglocator = `$cmd`; die "system $cmd failed: $?" if $?; + chomp($loglocator); $local_logfile = undef; # the temp file is automatically deleted Log (undef, "log manifest is $loglocator");