fixes and docs for testing crunch jobs locally

diff --git a/doc/user/api-tokens.md b/doc/user/api-tokens.md
index 582b3d507c515e787419073fe6f1c325e610b157..aa41060e0e8d242eda36cc4405e733679fb3ee83 100644 (file)
--- a/doc/user/api-tokens.md
+++ b/doc/user/api-tokens.md
@@ -16,7 +16,14 @@ Log in, if you haven't done that already.
 
 Click the "API tokens" link.
 
 
 Click the "API tokens" link.
 

-Copy an API token and set an environment variable in your terminal
+Copy an API token and set environment variables in your terminal

 session like this.
 
     export ARVADOS_API_TOKEN=unvz7ktg5p5k2q4wb9hpfl9fkge96rvv1j1gjpiq
 session like this.
 
     export ARVADOS_API_TOKEN=unvz7ktg5p5k2q4wb9hpfl9fkge96rvv1j1gjpiq

+    export ARVADOS_API_HOST=a123z.arvados.org
+
+If you are using a local development server with a self-signed
+certificate, you might need to bypass certificate verification. Don't
+do this if you are using a production service.
+
+    export ARVADOS_API_HOST_INSECURE=yes

diff --git a/doc/user/intro-jobs.textile b/doc/user/intro-jobs.textile
index 032ebf9950f14d99356d3dbe7d521fb69a25255a..98a37d7b34728055059a3d2030c8d94474c7dbba 100644 (file)
--- a/doc/user/intro-jobs.textile
+++ b/doc/user/intro-jobs.textile
@@ -39,15 +39,21 @@ h3. Developing and testing job scripts
 
 Usually, it makes sense to test your job script locally on small data sets.  When you are satisfied that it works, commit it to the git repository and run it in Arvados.
 
 
 Usually, it makes sense to test your job script locally on small data sets.  When you are satisfied that it works, commit it to the git repository and run it in Arvados.
 

-Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@
+Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@.
+
+Make sure you have @ARVADOS_API_TOKEN@ and @ARVADOS_API_HOST@ set correctly ("more info":api-tokens.html).

 
 Test your function:
 
 <pre>
 
 Test your function:
 
 <pre>

-read -rd "\000" newjob <<EOF; /path/to/arvados/services/crunch/crunch-job --job $newjob
+arvados=/path/to/arvados
+script_name=foo
+repo_path=/path/to/repo
+
+read -rd "\000" newjob <<EOF; $arvados/services/crunch/crunch-job --job "$newjob"

 {
 {

- "script":"foo",
- "script_version":"/path/to/working-copy",
+ "script":"$script_name",
+ "script_version":"$repo_path",

  "script_parameters":
  {
   "input":"..."
  "script_parameters":
  {
   "input":"..."


@@ -65,3 +71,17 @@ The @--job@ argument to @crunch-job@ is the same as the @--job@ argument to @arv
 * node resource constraints are ignored.
 
 You will see the progress of the job in your terminal.  Press Control-C to create a checkpoint and stop the job.
 * node resource constraints are ignored.
 
 You will see the progress of the job in your terminal.  Press Control-C to create a checkpoint and stop the job.

+
+h3. Testing job scripts without SDKs and Keep access
+
+Read and write data to /tmp/ instead of Keep. This only works with the Python SDK.
+
+<pre>
+export KEEP_LOCAL_STORE=/tmp
+</pre>
+
+Use the Perl SDK libraries directly from the arvados source tree.
+
+<pre>
+export PERLLIB=/path/to/arvados/sdk/perl/lib
+</pre>

diff --git a/sdk/python/arvados.py b/sdk/python/arvados.py
index 37ca4c8199282fb4ba93fd119b5b4cd6d41209d4..0ae5ad0c06df9c26ccccb455af4794200a155ade 100644 (file)
--- a/sdk/python/arvados.py
+++ b/sdk/python/arvados.py
@@ -130,9 +130,9 @@ class util:
             path = os.path.join(current_job().tmpdir, path)
         if not os.path.exists(path):
             util.run_command(["git", "clone", url, path],
             path = os.path.join(current_job().tmpdir, path)
         if not os.path.exists(path):
             util.run_command(["git", "clone", url, path],

-                             cwd=os.path.dirname(parser_path))
+                             cwd=os.path.dirname(path))

         util.run_command(["git", "checkout", version],
         util.run_command(["git", "checkout", version],

-                         cwd=parser_path)
+                         cwd=path)

         return path
 
 class DataReader:
         return path
 
 class DataReader:

diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index f61d33c7266ec2ebda89ca2bf9f9d41e4db0345c..e2fc19e852062b4f42267f3ebf8806c1205b3de9 100644 (file)
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -50,6 +50,12 @@ class Job < ArvadosModel
   protected
 
   def ensure_script_version_is_commit
   protected
 
   def ensure_script_version_is_commit

+    if self.is_locked_by and self.started_at
+      # Apparently client has already decided to go for it. This is
+      # needed to run a local job using a local working directory
+      # instead of a commit-ish.
+      return true
+    end

     sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
     if sha1
       self.script_version = sha1
     sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
     if sha1
       self.script_version = sha1

diff --git a/services/crunch/crunch-job b/services/crunch/crunch-job
index 30bc668280905d0bbab89e011f40c37a0096b376..710fc0b0c8caa404eaa19dc865270f6ef6bd3e74 100755 (executable)
--- a/services/crunch/crunch-job
+++ b/services/crunch/crunch-job
@@ -97,6 +97,7 @@ if (defined $job_api_token) {
 
 my $have_slurm = exists $ENV{SLURM_JOBID} && exists $ENV{SLURM_NODELIST};
 my $job_has_uuid = $jobspec =~ /^[-a-z\d]+$/;
 
 my $have_slurm = exists $ENV{SLURM_JOBID} && exists $ENV{SLURM_NODELIST};
 my $job_has_uuid = $jobspec =~ /^[-a-z\d]+$/;

+my $local_job = !$job_has_uuid;

 
 
 $SIG{'HUP'} = sub
 
 
 $SIG{'HUP'} = sub


@@ -119,7 +120,8 @@ my $metastream = Warehouse::Stream->new(whc => new Warehouse);
 $metastream->clear;
 $metastream->write_start('log.txt');
 
 $metastream->clear;
 $metastream->write_start('log.txt');
 

-my $User = {};
+my $User = $arv->{'users'}->{'current'}->execute;
+

 my $Job = {};
 my $job_id;
 my $dbh;
 my $Job = {};
 my $job_id;
 my $dbh;


@@ -127,7 +129,6 @@ my $sth;
 if ($job_has_uuid)
 {
   $Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
 if ($job_has_uuid)
 {
   $Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);

-  $User = $arv->{'users'}->{'current'}->execute;

   if (!$force_unlock) {
     if ($Job->{'is_locked_by'}) {
       croak("Job is locked: " . $Job->{'is_locked_by'});
   if (!$force_unlock) {
     if ($Job->{'is_locked_by'}) {
       croak("Job is locked: " . $Job->{'is_locked_by'});


@@ -153,11 +154,12 @@ else
     qw(script script_version script_parameters);
   }
 
     qw(script script_version script_parameters);
   }
 

-  if (!defined $Job->{'uuid'}) {
-    my $hostname = `hostname -s`;
-    chomp $hostname;
-    $Job->{'uuid'} = sprintf ("%s-t%d-p%d", $hostname, time, $$);
-  }
+  $Job->{'is_locked_by'} = $User->{'uuid'};
+  $Job->{'started_at'} = gmtime;
+
+  $Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job);
+
+  $job_has_uuid = 1;

 }
 $job_id = $Job->{'uuid'};
 
 }
 $job_id = $Job->{'uuid'};
 


@@ -314,21 +316,21 @@ else
 
 
 my $build_script;
 
 
 my $build_script;

-do {
-  local $/ = undef;
-  $build_script = <DATA>;
-};

 
 
 $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
 
 
 
 $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
 

-my $skip_install = (!$job_has_uuid && $Job->{script_version} =~ m{^/});
+my $skip_install = ($local_job && $Job->{script_version} =~ m{^/});

 if ($skip_install)
 {
   $ENV{"CRUNCH_SRC"} = $Job->{script_version};
 }
 else
 {
 if ($skip_install)
 {
   $ENV{"CRUNCH_SRC"} = $Job->{script_version};
 }
 else
 {

+  do {
+    local $/ = undef;
+    $build_script = <DATA>;
+  };

   Log (undef, "Install revision ".$Job->{script_version});
   my $nodelist = join(",", @node);
 
   Log (undef, "Install revision ".$Job->{script_version});
   my $nodelist = join(",", @node);
 


@@ -556,7 +558,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
     my @execargs = qw(sh);
     my $build_script_to_send = "";
     my $command =
     my @execargs = qw(sh);
     my $build_script_to_send = "";
     my $command =

-       "mkdir -p $ENV{CRUNCH_TMP}/revision "
+       "mkdir -p $ENV{CRUNCH_WORK} $ENV{CRUNCH_TMP} "

        ."&& cd $ENV{CRUNCH_TMP} ";
     if ($build_script)
     {
        ."&& cd $ENV{CRUNCH_TMP} ";
     if ($build_script)
     {