14198: Refactor and add support for --submit-runner-cluster
[arvados.git] / crunch_scripts / split-fastq.py
1 #!/usr/bin/python
2 # Copyright (C) The Arvados Authors. All rights reserved.
3 #
4 # SPDX-License-Identifier: Apache-2.0
5
6 import arvados
7 import re
8 import hashlib
9 import string
10
11 api = arvados.api('v1')
12
13 piece = 0
14 manifest_text = ""
15
16 # Look for paired reads
17
18 inp = arvados.CollectionReader(arvados.getjobparam('reads'))
19
20 manifest_list = []
21
22 def nextline(reader, start):
23     n = -1
24     while True:
25         r = reader.readfrom(start, 128)
26         if r == '':
27             break
28         n = string.find(r, "\n")
29         if n > -1:
30             break
31         else:
32             start += 128
33     return n
34
35 prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
36
37 # Look for fastq files
38 for s in inp.all_streams():
39     for f in s.all_files():
40         name_pieces = prog.match(f.name())
41         if name_pieces is not None:
42             if s.name() != ".":
43                 # The downstream tool (run-command) only iterates over the top
44                 # level of directories so if there are fastq files in
45                 # directories in the input, the choice is either to forget
46                 # there are directories (which might lead to name conflicts) or
47                 # just fail.
48                 print >>sys.stderr, "fastq must be at the root of the collection"
49                 sys.exit(1)
50
51             p = None
52             if name_pieces.group(2) is not None:
53                 if name_pieces.group(2) == "_1":
54                     p = [{}, {}]
55                     p[0]["reader"] = s.files()[name_pieces.group(0)]
56                     p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
57             else:
58                 p = [{}]
59                 p[0]["reader"] = s.files()[name_pieces.group(0)]
60
61             if p is not None:
62                 for i in xrange(0, len(p)):
63                     m = p[i]["reader"].as_manifest().split()
64                     m[0] = "./_" + str(piece)
65                     manifest_list.append(m)
66                 piece += 1
67
68 manifest_text = "\n".join(" ".join(m) for m in manifest_list) + "\n"
69
70 arvados.current_task().set_output(manifest_text)