From 8e4aa2b8873d8398b354f77457544ec5c810d30b Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 31 Oct 2013 21:41:59 -0700 Subject: [PATCH] allow crunch-job caller to override CRUNCH_TMP. fixes #1593 --- doc/user/intro-jobs.textile | 8 ++++++++ sdk/cli/bin/crunch-job | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/user/intro-jobs.textile b/doc/user/intro-jobs.textile index ccaa5a28ae..de548f08f7 100644 --- a/doc/user/intro-jobs.textile +++ b/doc/user/intro-jobs.textile @@ -71,6 +71,14 @@ The @--job@ argument to @arv-crunch-job@ is the same as the @--job@ argument to You will see the progress of the job in your terminal. Press Control-C to create a checkpoint and stop the job. +h3. Location of temporary files + +Crunch job tasks are supplied with @TASK_WORK@ and @JOB_WORK@ environment variables, to be used as scratch space. When running in local development mode, Crunch puts these in a directory called @crunch-job-{USERID}@ in @TMPDIR@ (or @/tmp@ if @TMPDIR@ is not set). + +* Set @TMPDIR@ to @/scratch@ to make Crunch use a directory like @/scratch/crunch-job-{USERID}/@ for temporary space. + +* Set @CRUNCH_TMP@ to @/scratch/foo@ to make Crunch use @/scratch/foo/@ for temporary space (omitting its customary @crunch-job-{USERID}@ part) + h3. Testing job scripts without SDKs and Keep access Read and write data to @/tmp/@ instead of Keep. This only works with the Python SDK. diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index e10df2a7ec..c4a58faa88 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -75,10 +75,12 @@ use Warehouse::Stream; use IPC::System::Simple qw(capturex); $ENV{"TMPDIR"} ||= "/tmp"; -$ENV{"CRUNCH_TMP"} = $ENV{"TMPDIR"} . "/crunch-job"; -if ($ENV{"USER"} ne "crunch" && $< != 0) { - # use a tmp dir unique for my uid - $ENV{"CRUNCH_TMP"} .= "-$<"; +unless (defined $ENV{"CRUNCH_TMP"}) { + $ENV{"CRUNCH_TMP"} = $ENV{"TMPDIR"} . "/crunch-job"; + if ($ENV{"USER"} ne "crunch" && $< != 0) { + # use a tmp dir unique for my uid + $ENV{"CRUNCH_TMP"} .= "-$<"; + } } $ENV{"JOB_WORK"} = $ENV{"CRUNCH_TMP"} . "/work"; $ENV{"CRUNCH_WORK"} = $ENV{"JOB_WORK"}; # deprecated -- 2.39.5