#!/usr/bin/env python3
import cgi, sys
sys.path.insert(0, '/opt/ngon/apps')
sys.path.insert(0, '/var/www/html/ngon')
from managers.auth_manager import AuthManager, generate_login_page_html
from links import generate_dropdown_html, generate_dropdown_css, generate_dropdown_js
_form = cgi.FieldStorage()
_auth = AuthManager('gen_outages')
_auth_required, _should_exit, _headers = _auth.require_auth(_form)
if _should_exit:
print("Content-Type: application/json")
if _headers:
print(_headers)
print("")
if _auth_required:
print('{"success": false, "error": "Authentication required"}')
else:
print('{"success": true}')
sys.exit(0)
if _auth_required:
print("Content-Type: text/html")
print("")
print(generate_login_page_html("Gen Outage Report"))
sys.exit(0)
_user_access = AuthManager.get_user_access()
"""
Gen Outage Report — finds problem gens that initiate group cascades.
Walks generator_alerts.log, classifies each fresh Down transition as:
- Initiator: at least N other gens in the same group went Down within WINDOW
- Follower: this Down was within WINDOW of an initiator's trip
- Lonely: Down with no sibling trips in window
Site-wide events (multiple groups at same site cascading within ~10 min) are
detected and flagged separately so external causes (gas line / weather / power)
don't pollute the per-gen problem-child stats.
Data: /opt/ngon/logs/generator_alerts.log + master_config gen→group mapping.
"""
import json
import os
import glob
import gzip
from collections import defaultdict
from datetime import datetime, timedelta
ALERTS_LOG = '/opt/ngon/logs/generator_alerts.log'
MASTER_CONFIG = '/opt/ngon/config/master_config.json'
# Cascade detection knobs
WINDOW_SECONDS = 120 # follower must trip within this many seconds of initiator
CASCADE_THRESHOLD = 2 # need at least this many sibling trips to count as cascade
SITE_WIDE_WINDOW_SECONDS = 600 # if cascades in 2+ groups at same site within this, mark site-wide
def parse_ts(s):
"""Handle both ISO (2025-10-23T17:19:15.816333) and space (2026-05-03 01:48:52) formats."""
s = s.strip()
try:
return datetime.fromisoformat(s)
except ValueError:
try:
return datetime.strptime(s, '%Y-%m-%d %H:%M:%S')
except ValueError:
return None
def load_gen_lookup():
"""Build gen_id → (site_name, group_name) map from master_config."""
lookup = {}
try:
with open(MASTER_CONFIG) as f:
cfg = json.load(f)
for site_name, site_data in cfg.get('sites', {}).items():
for group_name, group_data in site_data.get('generator_groups', {}).items():
for gen_id in group_data.get('generators', {}).keys():
lookup[gen_id] = (site_name, group_name)
except Exception:
pass
return lookup
def _alert_log_files():
"""All available alert-log files: live + rotated + gzipped.
logrotate (/etc/logrotate.d/ngon) rotates generator_alerts.log daily with
`rotate 14` + delaycompress, so the live file only ever holds today. The
history lives in generator_alerts.log.1 (uncompressed) and .2..14.gz — read
them all or the page only ever sees the current day. Retention caps real
coverage at ~14 days regardless of the selected window.
"""
files = []
if os.path.exists(ALERTS_LOG):
files.append(ALERTS_LOG)
files.extend(sorted(glob.glob(ALERTS_LOG + '.*')))
return files
def load_down_events(days):
"""Return list of fresh Down transitions: [(ts, gen_id, power_at_trip), ...]
sorted by timestamp ascending, across the live + rotated alert logs."""
cutoff = datetime.now() - timedelta(days=days)
events = []
for path in _alert_log_files():
try:
opener = gzip.open if path.endswith('.gz') else open
with opener(path, 'rt', errors='replace') as f:
for line in f:
parts = line.strip().split(',')
if len(parts) < 6:
continue
ts = parse_ts(parts[0])
if ts is None or ts < cutoff:
continue
gen_id = parts[1].strip()
alert_level = parts[2].strip()
power_str = parts[3].strip()
prev_alert = parts[5].strip()
# Only fresh Down transitions
if alert_level != 'Down' or prev_alert == 'Down':
continue
try:
power = float(power_str)
except (ValueError, TypeError):
power = 0.0
events.append((ts, gen_id, power))
except (FileNotFoundError, OSError):
continue
events.sort(key=lambda e: e[0])
return events
def classify_events(events, gen_lookup):
"""For each fresh Down event, decide initiator / follower / lonely.
Also detect site-wide cascades (multi-group within SITE_WIDE_WINDOW).
Returns:
cascades: [{initiator, initiator_ts, group, site, followers,
is_site_wide, power_at_trip}, ...]
lonely: list of (ts, gen_id, group, site, power)
"""
cascades = []
lonely = []
used_as_follower = set() # event indices already attributed to a cascade
for i, (ts, gen_id, power) in enumerate(events):
if i in used_as_follower:
continue
sg = gen_lookup.get(gen_id)
if not sg:
continue
site, group = sg
followers = [] # event indices of followers
for j in range(i + 1, len(events)):
f_ts, f_gen, _ = events[j]
if (f_ts - ts).total_seconds() > WINDOW_SECONDS:
break
if f_gen == gen_id:
continue
f_sg = gen_lookup.get(f_gen)
if f_sg and f_sg == sg: # same group
followers.append(j)
if len(followers) >= CASCADE_THRESHOLD:
cascades.append({
'initiator': gen_id,
'initiator_ts': ts,
'site': site,
'group': group,
'followers': [events[j][1] for j in followers],
'follower_idxs': followers,
'power_at_trip': power,
'is_site_wide': False,
})
used_as_follower.update(followers)
else:
lonely.append((ts, gen_id, group, site, power))
# Detect site-wide events: multiple cascades at same site within window.
# Group cascades by site and walk the sorted list looking for clusters
# spanning 2+ distinct groups within SITE_WIDE_WINDOW_SECONDS.
by_site = defaultdict(list)
for c in cascades:
by_site[c['site']].append(c)
for site, site_cascades in by_site.items():
site_cascades.sort(key=lambda c: c['initiator_ts'])
for i, c in enumerate(site_cascades):
window_groups = {c['group']}
window_indices = [i]
for j in range(i + 1, len(site_cascades)):
d = site_cascades[j]
if (d['initiator_ts'] - c['initiator_ts']).total_seconds() > SITE_WIDE_WINDOW_SECONDS:
break
window_groups.add(d['group'])
window_indices.append(j)
if len(window_groups) >= 2:
for k in window_indices:
site_cascades[k]['is_site_wide'] = True
return cascades, lonely
def build_gen_stats(events, cascades, lonely, gen_lookup):
"""Per-gen rollup: total outages, initiator count, follower count, last outage."""
stats = {}
# Total outages from raw event list
for ts, gen_id, power in events:
sg = gen_lookup.get(gen_id)
if not sg:
continue
site, group = sg
s = stats.setdefault(gen_id, {
'gen_id': gen_id, 'site': site, 'group': group,
'total': 0, 'initiated': 0, 'followed': 0, 'lonely': 0,
'initiated_excl_sitewide': 0,
'last_outage': None, 'last_power': 0.0,
})
s['total'] += 1
if s['last_outage'] is None or ts > s['last_outage']:
s['last_outage'] = ts
s['last_power'] = power
for c in cascades:
s = stats.get(c['initiator'])
if s:
s['initiated'] += 1
if not c['is_site_wide']:
s['initiated_excl_sitewide'] += 1
for f_gen in c['followers']:
fs = stats.get(f_gen)
if fs:
fs['followed'] += 1
for ts, gen_id, group, site, power in lonely:
s = stats.get(gen_id)
if s:
s['lonely'] += 1
return stats
def fmt_dt(dt):
if dt is None:
return '—'
return dt.strftime('%Y-%m-%d %H:%M')
def fmt_ago(dt):
if dt is None:
return '—'
delta = datetime.now() - dt
days = delta.days
if days >= 1:
return f"{days}d ago"
hours = delta.seconds // 3600
if hours >= 1:
return f"{hours}h ago"
mins = delta.seconds // 60
return f"{mins}m ago"
# ============================================================================
# Render
# ============================================================================
# logrotate (/etc/logrotate.d/ngon) keeps 14 days of alert logs, so windows
# beyond that silently show only what's retained. Cap the selector to match.
RETENTION_DAYS = 30
days_param = _form.getvalue('days', '14')
try:
days = max(1, min(RETENTION_DAYS, int(days_param)))
except ValueError:
days = RETENTION_DAYS
gen_lookup = load_gen_lookup()
events = load_down_events(days)
cascades, lonely = classify_events(events, gen_lookup)
stats = build_gen_stats(events, cascades, lonely, gen_lookup)
# Counts for header
total_events = len(events)
total_cascades = len(cascades)
total_site_wide = sum(1 for c in cascades if c['is_site_wide'])
total_group_cascades = total_cascades - total_site_wide
# Top initiators (excluding site-wide events)
top_initiators = sorted(
[s for s in stats.values() if s['initiated_excl_sitewide'] > 0],
key=lambda s: (-s['initiated_excl_sitewide'], -s['total']),
)[:10]
# Per-site/group rollup for the main table
sites = defaultdict(lambda: defaultdict(list)) # site -> group -> [stats]
for s in stats.values():
sites[s['site']][s['group']].append(s)
# Sort within each group
for site_groups in sites.values():
for group_list in site_groups.values():
group_list.sort(key=lambda s: (-s['initiated_excl_sitewide'], -s['total']))
# Recent cascades for the timeline section (most recent first)
recent_cascades = sorted(cascades, key=lambda c: c['initiator_ts'], reverse=True)[:50]
print("Content-Type: text/html")
print("")
# Build HTML pieces (kept out of f-string so braces don't fight)
top_rows_html = []
for s in top_initiators:
top_rows_html.append(
f"
"
f"
{s['gen_id']}
"
f"
{s['site']}
"
f"
{s['group']}
"
f"
{s['initiated_excl_sitewide']}
"
f"
{s['followed']}
"
f"
{s['total']}
"
f"
{fmt_ago(s['last_outage'])}
"
f"
"
)
top_rows = '\n'.join(top_rows_html) or "
No cascade initiators found in window.
"
site_sections_html = []
for site_name in sorted(sites.keys()):
group_blocks = []
for group_name in sorted(sites[site_name].keys()):
rows = []
for s in sites[site_name][group_name]:
init_class = 'red' if s['initiated_excl_sitewide'] > 0 else 'grey'
follow_class = 'orange' if s['followed'] > 0 else 'grey'
rows.append(
f"
"
f"
{s['gen_id']}
"
f"
{s['initiated_excl_sitewide']}
"
f"
{s['followed']}
"
f"
{s['lonely']}
"
f"
{s['total']}
"
f"
{fmt_dt(s['last_outage'])}
"
f"
{fmt_ago(s['last_outage'])}
"
f"
"
)
group_total = sum(s['total'] for s in sites[site_name][group_name])
group_init = sum(s['initiated_excl_sitewide'] for s in sites[site_name][group_name])
group_blocks.append(f"""
""")
site_sections = ''.join(site_sections_html) or "
No outages in selected window.
"
cascade_rows_html = []
for c in recent_cascades:
badge = "SITE-WIDE" if c['is_site_wide'] else "GROUP"
cascade_rows_html.append(
f"
"
f"
{fmt_dt(c['initiator_ts'])}
"
f"
{badge}
"
f"
{c['site']}
"
f"
{c['group']}
"
f"
{c['initiator']}
"
f"
{c['power_at_trip']:.0f} kW
"
f"
{', '.join(c['followers'])}
"
f"
"
)
cascade_rows = '\n'.join(cascade_rows_html) or "
No cascades in window.
"
print(f"""
Gen Outage Report
NGON Mining - Gen Outage Report
{generate_dropdown_html(_user_access)}
Generator outages and cascade-trigger analysis · identifies repeat-offender gens whose trips knock out their group
alert-log retention is {RETENTION_DAYS} days
Window:{days} days
Total outages:{total_events}
Group cascades:{total_group_cascades}
Site-wide events:{total_site_wide}
How to read:
A group cascade is when one gen trips and ≥{CASCADE_THRESHOLD} other gens in the same group also trip within {WINDOW_SECONDS}s.
The first gen down is classified as the Initiator; the rest are Followers.
A site-wide event is when cascades hit ≥2 different groups at the same site within {SITE_WIDE_WINDOW_SECONDS // 60} min — usually a shared external cause (gas, weather, transformer)
and excluded from the per-gen Initiator counts on this page so the problem-child stats stay focused on actionable failures.
Lonely = a Down with no sibling trips in window (single-gen failure, group survived).
Initiated vs Followed Others:Initiated counts cascades this gen started — its trip dragged the group down. High = root-cause troublemaker.
Followed Others counts cascades where a sibling tripped first and this gen fell along with the rest. High = structural weakness; gen survives standalone but loses load-redistribution races.
A gen high in both is a top swap candidate; high in only one tells you whether the failure mode is "this gen is the trigger" or "this gen can't take the hit when others fail."
Top Cascade Initiators
Gens whose trips have triggered group cascades. Site-wide events excluded. Repeat offenders here are candidates for swap or focused diagnostics.
Gen ID
Site
Group
Cascades Initiated
Followed Others
Total Outages
Last Outage
{top_rows}
All Outages by Site & Group
Per-gen breakdown. Sorted within each group by initiator count, then total outages.
{site_sections}
Recent Cascade Timeline
Last 50 detected cascades, most recent first. Site-wide events shown dimmed (not actionable at gen level).