projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
5493: add test for the getting_started help menu item.
[arvados.git]
/
crunch_scripts
/
decompress-all.py
diff --git
a/crunch_scripts/decompress-all.py
b/crunch_scripts/decompress-all.py
index 0566ffbe9e73f4df7d9382b6697af1332df87897..50d11f4d97440e331175ef78ac532b32e43a25e9 100755
(executable)
--- a/
crunch_scripts/decompress-all.py
+++ b/
crunch_scripts/decompress-all.py
@@
-16,6
+16,9
@@
import arvados
import re
import subprocess
import arvados
import re
import subprocess
+import os
+import sys
+import crunchutil.robust_put as robust_put
arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
input_as_path=True)
arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
input_as_path=True)
@@
-24,30
+27,35
@@
task = arvados.current_task()
input_file = task['parameters']['input']
input_file = task['parameters']['input']
-
result = re.match(r"(^[a-f0-9]{32}\+\d+)(\+\S+)*(/.*)(/.*)?
$", input_file)
+
infile_parts = re.match(r"(^[a-f0-9]{32}\+\d+)(\+\S+)*(/.*)?(/[^/]+)
$", input_file)
outdir = os.path.join(task.tmpdir, "output")
outdir = os.path.join(task.tmpdir, "output")
-os.m
k
dirs(outdir)
+os.m
ake
dirs(outdir)
os.chdir(outdir)
os.chdir(outdir)
-if result != None:
- cr = arvados.CollectionReader(re.group(1))
- streamname = '.'
- if re.group(3) != None:
- streamname += re.group(2)
- filename = re.group(3)[1:]
- else:
- filename = re.group(2)[1:]
+if infile_parts is None:
+ print >>sys.stderr, "Failed to parse input filename '%s' as a Keep file\n" % input_file
+ sys.exit(1)
- os.mkdirs(streamname)
+cr = arvados.CollectionReader(infile_parts.group(1))
+streamname = infile_parts.group(3)[1:]
+filename = infile_parts.group(4)[1:]
+
+if streamname is not None:
+ subprocess.call(["mkdir", "-p", streamname])
os.chdir(streamname)
os.chdir(streamname)
- streamreader = filter(lambda s: s.name() == streamname, cr.all_streams())[0]
- filereader = stream.files()[filename]
- rc = subprocess.call("dtrx", "-r", "-n", arvados.get_task_param_mount('input'))
+else:
+ streamname = '.'
+
+m = re.match(r'.*\.(gz|Z|bz2|tgz|tbz|zip|rar|7z|cab|deb|rpm|cpio|gem)$', arvados.get_task_param_mount('input'), re.IGNORECASE)
+
+if m is not None:
+ rc = subprocess.call(["dtrx", "-r", "-n", "-q", arvados.get_task_param_mount('input')])
if rc == 0:
if rc == 0:
- out.write_directory_tree(outdir, max_manifest_depth=0)
- arvados.task_set_output(out.finish())
+ task.set_output(robust_put.upload(outdir))
else:
else:
-
arvados.task_set_output(streamname + filereader.as_manifest()[1:]
)
+
sys.exit(rc
)
else:
else:
- sys.exit(1)
+ streamreader = filter(lambda s: s.name() == streamname, cr.all_streams())[0]
+ filereader = streamreader.files()[filename]
+ task.set_output(streamname + filereader.as_manifest()[1:])