crunch_scripts/grep

   1 #!/usr/bin/env python
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: Apache-2.0
   5
   6 import arvados
   7 import re
   8
   9 arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True)
  10
  11 this_job = arvados.current_job()
  12 this_task = arvados.current_task()
  13 this_task_input = this_task['parameters']['input']
  14 pattern = re.compile(this_job['script_parameters']['pattern'])
  15
  16 input_file = list(arvados.CollectionReader(this_task_input).all_files())[0]
  17 out = arvados.CollectionWriter()
  18 out.set_current_file_name(input_file.decompressed_name())
  19 out.set_current_stream_name(input_file.stream_name())
  20 for line in input_file.readlines():
  21     if pattern.search(line):
  22         out.write(line)
  23
  24 this_task.set_output(out.finish())