X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9f1850a385ee8e0a011474de19ee6507b0b168f3..528982e8b2d20552aefda334f49cc38469547294:/tools/crunchstat-summary/crunchstat_summary/reader.py diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py index 8ccdbc2fcf..2af770f51b 100644 --- a/tools/crunchstat-summary/crunchstat_summary/reader.py +++ b/tools/crunchstat-summary/crunchstat_summary/reader.py @@ -4,6 +4,7 @@ import arvados import itertools +import json import queue import threading @@ -11,24 +12,26 @@ from crunchstat_summary import logger class CollectionReader(object): - def __init__(self, collection_id): + def __init__(self, collection_id, api_client=None): self._collection_id = collection_id self._label = collection_id self._readers = [] + self._api_client = api_client + self._collection = arvados.collection.CollectionReader(self._collection_id, api_client=self._api_client) def __str__(self): return self._label def __iter__(self): logger.debug('load collection %s', self._collection_id) - collection = arvados.collection.CollectionReader(self._collection_id) - filenames = [filename for filename in collection] + + filenames = [filename for filename in self._collection] # Crunch2 has multiple stats files if len(filenames) > 1: filenames = ['crunchstat.txt', 'arv-mount.txt'] for filename in filenames: try: - self._readers.append(collection.open(filename)) + self._readers.append(self._collection.open(filename)) except IOError: logger.warn('Unable to open %s', filename) self._label = "{}/{}".format(self._collection_id, filenames[0]) @@ -43,6 +46,14 @@ class CollectionReader(object): reader.close() self._readers = [] + def node_info(self): + try: + with self._collection.open("node.json") as f: + return json.load(f) + except IOError: + logger.warn('Unable to open node.json') + return {} + class LiveLogReader(object): EOF = None