X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/7f93b99e31c946a1a92dafcbcd1d0e36a710c32b..b3a016e9a47d453b5ae4d287d8b6eaafd69971df:/tools/crunchstat-summary/crunchstat_summary/summarizer.py diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py index 51f7e43e07..e962ced314 100644 --- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py +++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py @@ -129,13 +129,14 @@ class Summarizer(object): try: self.label = m.group('job_uuid') except IndexError: - self.label = 'container' - if m.group('category').endswith(':'): + self.label = 'label #1' + category = m.group('category') + if category.endswith(':'): # "stderr crunchstat: notice: ..." continue - elif m.group('category') in ('error', 'caught'): + elif category in ('error', 'caught'): continue - elif m.group('category') in ('read', 'open', 'cgroup', 'CID', 'Running'): + elif category in ('read', 'open', 'cgroup', 'CID', 'Running'): # "stderr crunchstat: read /proc/1234/net/dev: ..." # (old logs are less careful with unprefixed error messages) continue @@ -167,9 +168,9 @@ class Summarizer(object): if task.finishtime is None or timestamp > task.finishtime: task.finishtime = timestamp - if self.starttime is None or timestamp < task.starttime: + if self.starttime is None or timestamp < self.starttime: self.starttime = timestamp - if self.finishtime is None or timestamp < task.finishtime: + if self.finishtime is None or timestamp > self.finishtime: self.finishtime = timestamp if (not self.detected_crunch1) and task.starttime is not None and task.finishtime is not None: @@ -207,24 +208,25 @@ class Summarizer(object): stats['user+sys'] = stats.get('user', 0) + stats.get('sys', 0) if 'tx' in stats or 'rx' in stats: stats['tx+rx'] = stats.get('tx', 0) + stats.get('rx', 0) - for stat, val in stats.items(): - if group == 'interval': - if stat == 'seconds': - this_interval_s = val - continue - elif not (this_interval_s > 0): + if group == 'interval': + if 'seconds' in stats: + this_interval_s = stats.get('seconds',0) + del stats['seconds'] + if this_interval_s <= 0: logger.error( "BUG? interval stat given with duration {!r}". format(this_interval_s)) - continue - else: + else: + logger.error('BUG? interval stat missing duration') + for stat, val in stats.items(): + if group == 'interval' and this_interval_s: stat = stat + '__rate' val = val / this_interval_s - if stat in ['user+sys__rate', 'tx+rx__rate']: + if stat in ['user+sys__rate', 'user__rate', 'sys__rate', 'tx+rx__rate', 'rx__rate', 'tx__rate']: task.series[category, stat].append( (timestamp - self.starttime, val)) else: - if stat in ['rss']: + if stat in ['rss','used','total']: task.series[category, stat].append( (timestamp - self.starttime, val)) self.task_stats[task_id][category][stat] = val @@ -314,7 +316,13 @@ class Summarizer(object): (float(self.job_tot['blkio:0:0']['read']) / float(self.job_tot['net:keep0']['rx'])) if self.job_tot['net:keep0']['rx'] > 0 else 0, - lambda x: x * 100.0)): + lambda x: x * 100.0), + ('Temp disk utilization {}%', + (float(self.job_tot['statfs']['used']) / + float(self.job_tot['statfs']['total'])) + if self.job_tot['statfs']['total'] > 0 else 0, + lambda x: x * 100.0), + ): format_string, val, transform = args if val == float('-Inf'): continue @@ -323,31 +331,40 @@ class Summarizer(object): yield "# "+format_string.format(self._format(val)) def _recommend_gen(self): + # TODO recommend fixing job granularity if elapsed time is too short return itertools.chain( self._recommend_cpu(), self._recommend_ram(), - self._recommend_keep_cache()) + self._recommend_keep_cache(), + self._recommend_temp_disk(), + ) def _recommend_cpu(self): """Recommend asking for 4 cores if max CPU usage was 333%""" constraint_key = self._map_runtime_constraint('vcpus') cpu_max_rate = self.stats_max['cpu']['user+sys__rate'] - if cpu_max_rate == float('-Inf'): + if cpu_max_rate == float('-Inf') or cpu_max_rate == 0.0: logger.warning('%s: no CPU usage data', self.label) return + # TODO Don't necessarily want to recommend on isolated max peak + # take average CPU usage into account as well or % time at max used_cores = max(1, int(math.ceil(cpu_max_rate))) asked_cores = self.existing_constraints.get(constraint_key) - if asked_cores is None or used_cores < asked_cores: + if asked_cores is None: + asked_cores = 1 + # TODO: This should be more nuanced in cases where max >> avg + if used_cores < asked_cores: yield ( '#!! {} max CPU usage was {}% -- ' - 'try runtime_constraints "{}":{}' + 'try reducing runtime_constraints to "{}":{}' ).format( self.label, math.ceil(cpu_max_rate*100), constraint_key, int(used_cores)) + # FIXME: This needs to be updated to account for current nodemanager algorithms def _recommend_ram(self): """Recommend an economical RAM constraint for this job. @@ -391,11 +408,11 @@ class Summarizer(object): asked_mib = self.existing_constraints.get(constraint_key) nearlygibs = lambda mebibytes: mebibytes/AVAILABLE_RAM_RATIO/1024 - if asked_mib is None or ( - math.ceil(nearlygibs(used_mib)) < nearlygibs(asked_mib)): + if used_mib > 0 and (asked_mib is None or ( + math.ceil(nearlygibs(used_mib)) < nearlygibs(asked_mib))): yield ( '#!! {} max RSS was {} MiB -- ' - 'try runtime_constraints "{}":{}' + 'try reducing runtime_constraints to "{}":{}' ).format( self.label, int(used_mib), @@ -415,7 +432,7 @@ class Summarizer(object): if utilization < 0.8: yield ( '#!! {} Keep cache utilization was {:.2f}% -- ' - 'try runtime_constraints "{}":{} (or more)' + 'try doubling runtime_constraints to "{}":{} (or more)' ).format( self.label, utilization * 100.0, @@ -423,6 +440,21 @@ class Summarizer(object): math.ceil(asked_cache * 2 / self._runtime_constraint_mem_unit())) + def _recommend_temp_disk(self): + """Recommend decreasing temp disk if utilization < 50%""" + total = float(self.job_tot['statfs']['total']) + utilization = (float(self.job_tot['statfs']['used']) / total) if total > 0 else 0.0 + + if utilization < 50.8 and total > 0: + yield ( + '#!! {} max temp disk utilization was {:.0f}% of {:.0f} MiB -- ' + 'consider reducing "tmpdirMin" and/or "outdirMin"' + ).format( + self.label, + utilization * 100.0, + total / MB) + + def _format(self, val): """Return a string representation of a stat.