10472: Conserve FDs by not opening collections until ready to read.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index a0a838d6c9538c6231e4d284b6a2d2e5b7df77d6..c215228ff5dbca1a0a92e376449adc45c6d548e9 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -14,21 +14,32 @@ from crunchstat_summary import logger
 
 class CollectionReader(object):
     def __init__(self, collection_id):
-        logger.debug('load collection %s', collection_id)
-        collection = arvados.collection.CollectionReader(collection_id)
+        self._collection_id = collection_id
+        self._label = collection_id
+        self._reader = None
+
+    def __str__(self):
+        return self._label
+
+    def __iter__(self):
+        logger.debug('load collection %s', self._collection_id)
+        collection = arvados.collection.CollectionReader(self._collection_id)
         filenames = [filename for filename in collection]
         if len(filenames) == 1:
             filename = filenames[0]
         else:
             filename = 'crunchstat.txt'
+        self._label = "{}/{}".format(self._collection_id, filename)
         self._reader = collection.open(filename)
-        self._label = "{}/{}".format(collection_id, filename)
+        return iter(self._reader)
 
-    def __str__(self):
-        return self._label
+    def __enter__(self):
+        return self
 
-    def __iter__(self):
-        return iter(self._reader)
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._reader:
+            self._reader.close()
+            self._reader = None
 
 
 class LiveLogReader(object):

diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 4924b400dd3d2db8a9e25301c10f7ff6f26c4a64..33b8db9af8fb2678fa68122c68b44383d2aa1386 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -70,8 +70,12 @@ class Summarizer(object):
 
     def run(self):
         logger.debug("%s: parsing logdata %s", self.label, self._logdata)
+        with self._logdata as logdata:
+            self._run(logdata)
+
+    def _run(self, logdata):
         self.detected_crunch1 = False
-        for line in self._logdata:
+        for line in logdata:
             if not self.detected_crunch1 and '-8i9sb-' in line:
                 self.detected_crunch1 = True