crunch_scripts/split-fastq.py

   1 #!/usr/bin/python
   2
   3 import arvados
   4 import re
   5 import hashlib
   6 import string
   7
   8 api = arvados.api('v1')
   9
  10 piece = 0
  11 manifest_text = ""
  12
  13 # Look for paired reads
  14
  15 inp = arvados.CollectionReader(arvados.getjobparam('reads'))
  16
  17 manifest_list = []
  18
  19 def nextline(reader, start):
  20     n = -1
  21     while True:
  22         r = reader.readfrom(start, 128)
  23         if r == '':
  24             break
  25         n = string.find(r, "\n")
  26         if n > -1:
  27             break
  28         else:
  29             start += 128
  30     return n
  31
  32 prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
  33
  34 # Look for fastq files
  35 for s in inp.all_streams():
  36     for f in s.all_files():
  37         name_pieces = prog.match(f.name())
  38         if name_pieces is not None:
  39             if s.name() != ".":
  40                 # The downstream tool (run-command) only iterates over the top
  41                 # level of directories so if there are fastq files in
  42                 # directories in the input, the choice is either to forget
  43                 # there are directories (which might lead to name conflicts) or
  44                 # just fail.
  45                 print >>sys.stderr, "fastq must be at the root of the collection"
  46                 sys.exit(1)
  47
  48             p = None
  49             if name_pieces.group(2) is not None:
  50                 if name_pieces.group(2) == "_1":
  51                     p = [{}, {}]
  52                     p[0]["reader"] = s.files()[name_pieces.group(0)]
  53                     p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
  54             else:
  55                 p = [{}]
  56                 p[0]["reader"] = s.files()[name_pieces.group(0)]
  57
  58             if p is not None:
  59                 for i in xrange(0, len(p)):
  60                     m = p[i]["reader"].as_manifest().split()
  61                     m[0] = "./_" + str(piece)
  62                     manifest_list.append(m)
  63                 piece += 1
  64
  65 manifest_text = "\n".join(" ".join(m) for m in manifest_list) + "\n"
  66
  67 arvados.current_task().set_output(manifest_text)