#!/usr/bin/python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
import arvados
import re
import hashlib
+import string
api = arvados.api('v1')
piece = 0
manifest_text = ""
-def put_manifest(manifest_text, sources=[]):
- crm = arvados.CollectionReader(manifest_text)
-
- combined = crm.manifest_text(strip=True)
-
- m = hashlib.new('md5')
- m.update(combined)
-
- print combined
-
- uuid = "{}+{}".format(m.hexdigest(), len(combined))
-
- collection = arvados.api().collections().create(
- body={
- 'uuid': uuid,
- 'manifest_text': crm.manifest_text(),
- }).execute()
-
- for s in sources:
- l = arvados.api().links().create(body={
- "link": {
- "tail_uuid": s,
- "head_uuid": uuid,
- "link_class": "provenance",
- "name": "provided"
- }}).execute()
-
- return uuid
-
# Look for paired reads
inp = arvados.CollectionReader(arvados.getjobparam('reads'))
-prog = re.compile("(.*?)_1.fastq(.gz)?$")
+manifest_list = []
-manifest_text = ""
+def nextline(reader, start):
+ n = -1
+ while True:
+ r = reader.readfrom(start, 128)
+ if r == '':
+ break
+ n = string.find(r, "\n")
+ if n > -1:
+ break
+ else:
+ start += 128
+ return n
+prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
+
+# Look for fastq files
for s in inp.all_streams():
- if s.name() == ".":
- for f in s.all_files():
- result = prog.match(f.name())
- if result != None:
- p = [{}, {}]
- p[0]["reader"] = s.files()[result.group(0)]
- p[1]["reader"] = s.files()[result.group(1) + "_2.fastq" + result.group(2)]
- m0 = p[0]["reader"].as_manifest()[1:]
- m1 = p[1]["reader"].as_manifest()[1:]
- manifest_text += "./_" + str(piece) + m0
- manifest_text += "./_" + str(piece) + m1
+ for f in s.all_files():
+ name_pieces = prog.match(f.name())
+ if name_pieces is not None:
+ if s.name() != ".":
+ # The downstream tool (run-command) only iterates over the top
+ # level of directories so if there are fastq files in
+ # directories in the input, the choice is either to forget
+ # there are directories (which might lead to name conflicts) or
+ # just fail.
+ print >>sys.stderr, "fastq must be at the root of the collection"
+ sys.exit(1)
+
+ p = None
+ if name_pieces.group(2) is not None:
+ if name_pieces.group(2) == "_1":
+ p = [{}, {}]
+ p[0]["reader"] = s.files()[name_pieces.group(0)]
+ p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
+ else:
+ p = [{}]
+ p[0]["reader"] = s.files()[name_pieces.group(0)]
+
+ if p is not None:
+ for i in xrange(0, len(p)):
+ m = p[i]["reader"].as_manifest().split()
+ m[0] = "./_" + str(piece)
+ manifest_list.append(m)
piece += 1
-# No pairs found so just put each fastq file into a separate directory
-if manifest_text == "":
- for s in inp.all_streams():
- prog = re.compile("(.*?).fastq(.gz)?$")
- if s.name() == ".":
- for f in s.all_files():
- result = prog.match(f.name())
- if result != None:
- p = [{}]
- p[0]["reader"] = s.files()[result.group(0)]
- m0 = p[0]["reader"].as_manifest()[1:]
- manifest_text += "./_" + str(piece) + m0
- piece += 1
-
-print manifest_text
+manifest_text = "\n".join(" ".join(m) for m in manifest_list) + "\n"
-arvados.current_task().set_output(put_manifest(manifest_text, [arvados.getjobparam('reads')]))
+arvados.current_task().set_output(manifest_text)
projects / arvados.git / blobdiff