From cdde7f246fec59bc99da86145fd4cf4efcf37a68 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 27 Jun 2013 16:06:32 -0400 Subject: [PATCH] fixes and docs for testing crunch jobs locally --- doc/user/api-tokens.md | 9 ++++++++- doc/user/intro-jobs.textile | 28 ++++++++++++++++++++++++---- sdk/python/arvados.py | 4 ++-- services/api/app/models/job.rb | 6 ++++++ services/crunch/crunch-job | 28 +++++++++++++++------------- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/doc/user/api-tokens.md b/doc/user/api-tokens.md index 582b3d507c..aa41060e0e 100644 --- a/doc/user/api-tokens.md +++ b/doc/user/api-tokens.md @@ -16,7 +16,14 @@ Log in, if you haven't done that already. Click the "API tokens" link. -Copy an API token and set an environment variable in your terminal +Copy an API token and set environment variables in your terminal session like this. export ARVADOS_API_TOKEN=unvz7ktg5p5k2q4wb9hpfl9fkge96rvv1j1gjpiq + export ARVADOS_API_HOST=a123z.arvados.org + +If you are using a local development server with a self-signed +certificate, you might need to bypass certificate verification. Don't +do this if you are using a production service. + + export ARVADOS_API_HOST_INSECURE=yes diff --git a/doc/user/intro-jobs.textile b/doc/user/intro-jobs.textile index 032ebf9950..98a37d7b34 100644 --- a/doc/user/intro-jobs.textile +++ b/doc/user/intro-jobs.textile @@ -39,15 +39,21 @@ h3. Developing and testing job scripts Usually, it makes sense to test your job script locally on small data sets. When you are satisfied that it works, commit it to the git repository and run it in Arvados. -Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@ +Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@. + +Make sure you have @ARVADOS_API_TOKEN@ and @ARVADOS_API_HOST@ set correctly ("more info":api-tokens.html). Test your function: 
-read -rd "\000" newjob <
+export KEEP_LOCAL_STORE=/tmp
+
+ +Use the Perl SDK libraries directly from the arvados source tree. + +
+export PERLLIB=/path/to/arvados/sdk/perl/lib
+
diff --git a/sdk/python/arvados.py b/sdk/python/arvados.py index 37ca4c8199..0ae5ad0c06 100644 --- a/sdk/python/arvados.py +++ b/sdk/python/arvados.py @@ -130,9 +130,9 @@ class util: path = os.path.join(current_job().tmpdir, path) if not os.path.exists(path): util.run_command(["git", "clone", url, path], - cwd=os.path.dirname(parser_path)) + cwd=os.path.dirname(path)) util.run_command(["git", "checkout", version], - cwd=parser_path) + cwd=path) return path class DataReader: diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb index f61d33c726..e2fc19e852 100644 --- a/services/api/app/models/job.rb +++ b/services/api/app/models/job.rb @@ -50,6 +50,12 @@ class Job < ArvadosModel protected def ensure_script_version_is_commit + if self.is_locked_by and self.started_at + # Apparently client has already decided to go for it. This is + # needed to run a local job using a local working directory + # instead of a commit-ish. + return true + end sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil if sha1 self.script_version = sha1 diff --git a/services/crunch/crunch-job b/services/crunch/crunch-job index 30bc668280..710fc0b0c8 100755 --- a/services/crunch/crunch-job +++ b/services/crunch/crunch-job @@ -97,6 +97,7 @@ if (defined $job_api_token) { my $have_slurm = exists $ENV{SLURM_JOBID} && exists $ENV{SLURM_NODELIST}; my $job_has_uuid = $jobspec =~ /^[-a-z\d]+$/; +my $local_job = !$job_has_uuid; $SIG{'HUP'} = sub @@ -119,7 +120,8 @@ my $metastream = Warehouse::Stream->new(whc => new Warehouse); $metastream->clear; $metastream->write_start('log.txt'); -my $User = {}; +my $User = $arv->{'users'}->{'current'}->execute; + my $Job = {}; my $job_id; my $dbh; @@ -127,7 +129,6 @@ my $sth; if ($job_has_uuid) { $Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec); - $User = $arv->{'users'}->{'current'}->execute; if (!$force_unlock) { if ($Job->{'is_locked_by'}) { croak("Job is locked: " . $Job->{'is_locked_by'}); @@ -153,11 +154,12 @@ else qw(script script_version script_parameters); } - if (!defined $Job->{'uuid'}) { - my $hostname = `hostname -s`; - chomp $hostname; - $Job->{'uuid'} = sprintf ("%s-t%d-p%d", $hostname, time, $$); - } + $Job->{'is_locked_by'} = $User->{'uuid'}; + $Job->{'started_at'} = gmtime; + + $Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job); + + $job_has_uuid = 1; } $job_id = $Job->{'uuid'}; @@ -314,21 +316,21 @@ else my $build_script; -do { - local $/ = undef; - $build_script = ; -}; $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version}; -my $skip_install = (!$job_has_uuid && $Job->{script_version} =~ m{^/}); +my $skip_install = ($local_job && $Job->{script_version} =~ m{^/}); if ($skip_install) { $ENV{"CRUNCH_SRC"} = $Job->{script_version}; } else { + do { + local $/ = undef; + $build_script = ; + }; Log (undef, "Install revision ".$Job->{script_version}); my $nodelist = join(",", @node); @@ -556,7 +558,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++) my @execargs = qw(sh); my $build_script_to_send = ""; my $command = - "mkdir -p $ENV{CRUNCH_TMP}/revision " + "mkdir -p $ENV{CRUNCH_WORK} $ENV{CRUNCH_TMP} " ."&& cd $ENV{CRUNCH_TMP} "; if ($build_script) { -- 2.30.2