12483: Add Rename(old,new) to CollectionFileSystem.

[arvados.git] / tools / crunchstat-summary / crunchstat_summary / summarizer.py

diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 6e08c5e1b0ecf7a2898d36cc8ca6d4e775c5f272..d91161c70c3aabdaa6223063946686f437adeaa0 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -132,9 +132,9 @@ class Summarizer(object):
             if m.group('category').endswith(':'):
                 # "stderr crunchstat: notice: ..."
                 continue
-            elif m.group('category') in ('error', 'caught', 'Running'):
+            elif m.group('category') in ('error', 'caught'):
                 continue
-            elif m.group('category') in ['read', 'open', 'cgroup', 'CID']:
+            elif m.group('category') in ('read', 'open', 'cgroup', 'CID', 'Running'):
                 # "stderr crunchstat: read /proc/1234/net/dev: ..."
                 # (old logs are less careful with unprefixed error messages)
                 continue
@@ -182,8 +182,10 @@ class Summarizer(object):
                         else:
                             stats[stat] = int(val)
                 except ValueError as e:
-                    logger.warning('Error parsing {} stat: {!r}'.format(
-                        stat, e))
+                    logger.warning(
+                        'Error parsing value %r (stat %r, category %r): %r',
+                        val, stat, category, e)
+                    logger.warning('%s', line)
                     continue
                 if 'user' in stats or 'sys' in stats:
                     stats['user+sys'] = stats.get('user', 0) + stats.get('sys', 0)
@@ -227,6 +229,8 @@ class Summarizer(object):
 
     def long_label(self):
         label = self.label
+        if hasattr(self, 'process') and self.process['uuid'] not in label:
+            label = '{} ({})'.format(label, self.process['uuid'])
         if self.finishtime:
             label += ' -- elapsed time '
             s = (self.finishtime - self.starttime).total_seconds()
@@ -528,10 +532,11 @@ class MultiSummarizer(object):
 
     def text_report(self):
         txt = ''
-        for cname, child in self._descendants():
-            if len(self.children) > 1:
+        d = self._descendants()
+        for child in d.itervalues():
+            if len(d) > 1:
                 txt += '### Summary for {} ({})\n'.format(
-                    cname, child.process['uuid'])
+                    child.label, child.process['uuid'])
             txt += child.text_report()
             txt += '\n'
         return txt
@@ -539,12 +544,13 @@ class MultiSummarizer(object):
     def _descendants(self):
         """Dict of self and all descendants.
 
-        Wrappers with nothing of their own to report are omitted.
+        Nodes with nothing of their own to report (like
+        MultiSummarizers) are omitted.
         """
         d = collections.OrderedDict()
-        for cname, child in self.children.iteritems():
+        for key, child in self.children.iteritems():
             if isinstance(child, Summarizer):
-                d[cname] = child
+                d[key] = child
             if isinstance(child, MultiSummarizer):
                 d.update(child._descendants())
         return d
@@ -557,14 +563,10 @@ class JobTreeSummarizer(MultiSummarizer):
     """Summarizes a job and all children listed in its components field."""
     def __init__(self, job, label=None, **kwargs):
         arv = arvados.api('v1', model=OrderedJsonModel())
-        label = label or job.get('name', None)
-        if label is None:
-            label = job['uuid']
-        else:
-            label = ' '.join([label, job['uuid']])
+        label = label or job.get('name', job['uuid'])
         children = collections.OrderedDict()
         children[job['uuid']] = JobSummarizer(job, label=label, **kwargs)
-        if job['components']:
+        if job.get('components', None):
             preloaded = {}
             for j in arv.jobs().index(
                     limit=len(job['components']),
@@ -592,7 +594,7 @@ class PipelineSummarizer(MultiSummarizer):
             else:
                 logger.info(
                     "%s: job %s", cname, component['job']['uuid'])
-                summarizer = JobTreeSummarizer(component['job'], **kwargs)
+                summarizer = JobTreeSummarizer(component['job'], label=cname, **kwargs)
                 summarizer.label = '{} {}'.format(
                     cname, component['job']['uuid'])
                 children[cname] = summarizer
@@ -603,7 +605,7 @@ class PipelineSummarizer(MultiSummarizer):
 
 
 class ContainerTreeSummarizer(MultiSummarizer):
-    def __init__(self, root, **kwargs):
+    def __init__(self, root, skip_child_jobs=False, **kwargs):
         arv = arvados.api('v1', model=OrderedJsonModel())
 
         label = kwargs.pop('label', None) or root.get('name') or root['uuid']
@@ -624,15 +626,20 @@ class ContainerTreeSummarizer(MultiSummarizer):
 
             page_filters = []
             while True:
-                items = arv.container_requests().index(
+                child_crs = arv.container_requests().index(
                     order=['uuid asc'],
                     filters=page_filters+[
                         ['requesting_container_uuid', '=', current['uuid']]],
-                ).execute()['items']
-                if not items:
+                ).execute()
+                if not child_crs['items']:
+                    break
+                elif skip_child_jobs:
+                    logger.warning('%s: omitting stats from %d child containers'
+                                   ' because --skip-child-jobs flag is on',
+                                   label, child_crs['items_available'])
                     break
-                page_filters = [['uuid', '>', items[-1]['uuid']]]
-                for cr in items:
+                page_filters = [['uuid', '>', child_crs['items'][-1]['uuid']]]
+                for cr in child_crs['items']:
                     if cr['container_uuid']:
                         logger.debug('%s: container req %s', current['uuid'], cr['uuid'])
                         cr['name'] = cr.get('name') or cr['uuid']