Click the "API tokens" link.
-Copy an API token and set an environment variable in your terminal
+Copy an API token and set environment variables in your terminal
session like this.
export ARVADOS_API_TOKEN=unvz7ktg5p5k2q4wb9hpfl9fkge96rvv1j1gjpiq
+ export ARVADOS_API_HOST=a123z.arvados.org
+
+If you are using a local development server with a self-signed
+certificate, you might need to bypass certificate verification. Don't
+do this if you are using a production service.
+
+ export ARVADOS_API_HOST_INSECURE=yes
Usually, it makes sense to test your job script locally on small data sets. When you are satisfied that it works, commit it to the git repository and run it in Arvados.
-Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@
+Save your job script (say, @foo@) in @{git-repo}/crunch_scripts/foo@.
+
+Make sure you have @ARVADOS_API_TOKEN@ and @ARVADOS_API_HOST@ set correctly ("more info":api-tokens.html).
Test your function:
<pre>
-read -rd "\000" newjob <<EOF; /path/to/arvados/services/crunch/crunch-job --job $newjob
+arvados=/path/to/arvados
+script_name=foo
+repo_path=/path/to/repo
+
+read -rd "\000" newjob <<EOF; $arvados/services/crunch/crunch-job --job "$newjob"
{
- "script":"foo",
- "script_version":"/path/to/working-copy",
+ "script":"$script_name",
+ "script_version":"$repo_path",
"script_parameters":
{
"input":"..."
* node resource constraints are ignored.
You will see the progress of the job in your terminal. Press Control-C to create a checkpoint and stop the job.
+
+h3. Testing job scripts without SDKs and Keep access
+
+Read and write data to /tmp/ instead of Keep. This only works with the Python SDK.
+
+<pre>
+export KEEP_LOCAL_STORE=/tmp
+</pre>
+
+Use the Perl SDK libraries directly from the arvados source tree.
+
+<pre>
+export PERLLIB=/path/to/arvados/sdk/perl/lib
+</pre>
path = os.path.join(current_job().tmpdir, path)
if not os.path.exists(path):
util.run_command(["git", "clone", url, path],
- cwd=os.path.dirname(parser_path))
+ cwd=os.path.dirname(path))
util.run_command(["git", "checkout", version],
- cwd=parser_path)
+ cwd=path)
return path
class DataReader:
protected
def ensure_script_version_is_commit
+ if self.is_locked_by and self.started_at
+ # Apparently client has already decided to go for it. This is
+ # needed to run a local job using a local working directory
+ # instead of a commit-ish.
+ return true
+ end
sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
if sha1
self.script_version = sha1
my $have_slurm = exists $ENV{SLURM_JOBID} && exists $ENV{SLURM_NODELIST};
my $job_has_uuid = $jobspec =~ /^[-a-z\d]+$/;
+my $local_job = !$job_has_uuid;
$SIG{'HUP'} = sub
$metastream->clear;
$metastream->write_start('log.txt');
-my $User = {};
+my $User = $arv->{'users'}->{'current'}->execute;
+
my $Job = {};
my $job_id;
my $dbh;
if ($job_has_uuid)
{
$Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
- $User = $arv->{'users'}->{'current'}->execute;
if (!$force_unlock) {
if ($Job->{'is_locked_by'}) {
croak("Job is locked: " . $Job->{'is_locked_by'});
qw(script script_version script_parameters);
}
- if (!defined $Job->{'uuid'}) {
- my $hostname = `hostname -s`;
- chomp $hostname;
- $Job->{'uuid'} = sprintf ("%s-t%d-p%d", $hostname, time, $$);
- }
+ $Job->{'is_locked_by'} = $User->{'uuid'};
+ $Job->{'started_at'} = gmtime;
+
+ $Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job);
+
+ $job_has_uuid = 1;
}
$job_id = $Job->{'uuid'};
my $build_script;
-do {
- local $/ = undef;
- $build_script = <DATA>;
-};
$ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
-my $skip_install = (!$job_has_uuid && $Job->{script_version} =~ m{^/});
+my $skip_install = ($local_job && $Job->{script_version} =~ m{^/});
if ($skip_install)
{
$ENV{"CRUNCH_SRC"} = $Job->{script_version};
}
else
{
+ do {
+ local $/ = undef;
+ $build_script = <DATA>;
+ };
Log (undef, "Install revision ".$Job->{script_version});
my $nodelist = join(",", @node);
my @execargs = qw(sh);
my $build_script_to_send = "";
my $command =
- "mkdir -p $ENV{CRUNCH_TMP}/revision "
+ "mkdir -p $ENV{CRUNCH_WORK} $ENV{CRUNCH_TMP} "
."&& cd $ENV{CRUNCH_TMP} ";
if ($build_script)
{