#!/usr/bin/python3 """ site_issues_dev.py — DEV/scratch page for the proactive "site issues" summary. Stage 1: dumps EVERY harvested candidate signal per site so we can eyeball the raw data and work through what becomes an issue vs what gets suppressed. Not fancy on purpose. Reuses page_key 'status' for auth (no new nav entry). Backend harvester: /opt/ngon/apps/monitoring/site_issues.py """ import cgi import sys import html import time sys.path.insert(0, '/opt/ngon/apps') from managers.auth_manager import AuthManager, generate_login_page_html from monitoring.site_issues import harvest, derive_issues form = cgi.FieldStorage() auth = AuthManager('status') auth_required, should_exit, headers = auth.require_auth(form) if should_exit: print("Content-Type: application/json") if headers: print(headers) print("") if auth_required: print('{"success": false, "error": "Authentication required"}') else: print('{"success": true}') sys.exit(0) if auth_required: print("Content-Type: text/html\n") print(generate_login_page_html("Site Issues (dev)")) sys.exit(0) def esc(v): if v is None: return "–" if isinstance(v, bool): return "yes" if v else "no" if isinstance(v, float): v = round(v, 1) return html.escape(str(v)) def cell(v, bad=False, warn=False): cls = " class='bad'" if bad else (" class='warn'" if warn else "") return f"{esc(v)}" def age_str(sec): if sec is None: return "–" sec = int(sec) if sec < 120: return f"{sec}s" if sec < 7200: return f"{sec // 60}m" if sec < 172800: return f"{sec // 3600}h" return f"{sec // 86400}d" data = harvest() issues = derive_issues(data) print("Content-Type: text/html\n") parts = [] parts.append(""" Site Issues (dev) """) gen_ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data.get("generated_at", time.time()))) parts.append(f"

Site Issues — raw signal dump (dev)

") parts.append(f"
generated {gen_ts} · auto-refresh 5 min · this is the everything-view we work through, not the final list
") for e in data.get("errors", []): parts.append(f"
harvest error: {esc(e)}
") # ---- MOCKUP: what the status-page "site issues" panel would show ---- parts.append("
") parts.append("

▸ Site Issues  (proposed status-page panel)

") parts.append("
This is the display. Rules so far: pod-down owns its offline miners · " "offline cluster only on online pods · no-bridge gens ignored · maintenance-tagged gens muted · " "sleeping NOT evaluated yet.
") def render_node(node): label = esc(node.get("label")) cnt = node.get("count") head = label + (f" ({cnt})" if cnt is not None else "") children = node.get("children") if children: h = [f"
{head}"] for c in children: h.append(render_node(c)) h.append("
") return "".join(h) h = [f"
{head}
"] for ln in node.get("lines") or []: h.append(f"
{esc(ln)}
") return "".join(h) for si in issues["sites"]: cnt = si["issue_count"] if cnt == 0: parts.append("
" f"{esc(si['name'])}" f"✓ clear
") continue parts.append(f"
" f"{esc(si['name'])}" f"{cnt}") for ln in si.get("network", []): parts.append(f"
• {esc(ln)}
") for cat in si.get("categories", []): unit = cat.get("unit") ctitle = (f"{esc(cat['name'])} " f"({cat['count']}{' ' + esc(unit) if unit else ''})") parts.append(f"
{ctitle}") for node in cat.get("nodes", []): parts.append(render_node(node)) parts.append("
") parts.append("
") for m in issues.get("muted_sites", []): parts.append("
" f"{esc(m['name'])}" f"muted" f"
known exception — {esc(m['reason'])}
") if issues.get("hidden_sites"): parts.append(f"
hidden as non-operational: " f"{esc(', '.join(issues['hidden_sites']))} — confirm this list
") parts.append("
muted sites & chronic-warning list are placeholders for the " "Site Manager context field (operator-authored exceptions the LLM will read).
") parts.append("
") parts.append("

Raw signal dump (everything harvested)

") for s in data["sites"]: # Site header badges badges = [] ps = s.get("peplink_status") if ps == "offline": badges.append("peplink offline") elif ps == "online": badges.append("peplink online") star = s.get("starlink_up") if star is False: badges.append("starlink DOWN") elif star is True: badges.append("starlink up") g = s.get("gas") or {} if g.get("flow_mcfd") is not None: badges.append(f"gas {esc(g.get('flow_mcfd'))} mcfd / {esc(g.get('sp_psia'))} psia · {age_str(g.get('age_sec'))}") parts.append("
") parts.append(f"
{esc(s['name'])} {''.join(badges)}
") for grp in s["groups"]: parts.append("
") gbadges = [] if grp.get("peplink_status") == "offline": gbadges.append("pep offline") if grp.get("starlink_up") is False: gbadges.append("starlink DOWN") parts.append(f"

{esc(grp['name'])}   " f"max {esc(grp.get('max_gen_kw'))}kW · " f"running {esc(grp.get('running'))} / down {esc(grp.get('down'))} " f"{''.join(gbadges)}

") # Gens table if grp["gens"]: parts.append("
") parts.append("" "" "") for gen in grp["gens"]: src = gen.get("data_source") live_side = src in ("live", "recent") kw = gen.get("live_power") if live_side else gen.get("mesa_power") psi = gen.get("live_ng_pressure") if live_side else gen.get("mesa_ng_pressure") run = gen.get("live_run_status") if live_side else None wl = gen.get("warning_level") wtags = [] for w in gen.get("warnings", []): cls = "wtag crit" if w.get("level") == "critical" else "wtag" lbl = w.get("label") or "?" if w.get("value") is not None: _u = w.get("unit") or "" lbl += f" {esc(w.get('value'))}{'' if _u in ('', '%') else ' '}{esc(_u)}" wtags.append(f"{lbl}") src_html = { "live": "", "recent": "", "mesa": "", }.get(src, f"") row = [""] row.append(cell(gen.get("id"))) row.append(src_html) row.append(cell(gen.get("status"), bad=(gen.get("status") == "Down"))) row.append(cell(gen.get("maintenance_status"), warn=bool(gen.get("maintenance_status")))) row.append(cell(kw)) row.append(cell(psi)) row.append(cell(run)) row.append(f"") row.append(cell(wl, bad=(wl == "critical"), warn=(wl == "warn"))) row.append(f"") row.append(cell(gen.get("down_reason"), warn=bool(gen.get("down_reason")))) row.append(cell(gen.get("battery"))) row.append(cell(gen.get("hold"), warn=bool(gen.get("hold")))) row.append("") parts.append("".join(row)) parts.append("
gensrcstatusmaintkWNG psirunlive-agewarnwarningsdown reasonbatthold
liverecent?mesa{esc(src)}
{age_str(gen.get('live_age_sec'))}{''.join(wtags) if wtags else '–'}
") # Pods table if grp["pods"]: parts.append("
") parts.append("" "" "") for p in grp["pods"]: offline = p.get("miners_offline") or 0 zero = p.get("miners_zero_hash") or 0 sleeping = p.get("miners_sleeping") or 0 conn_down = (p.get("conn") == "offline") row = [""] row.append(cell(p.get("name"))) row.append(cell(p.get("conn"), bad=conn_down)) row.append(cell(p.get("miner_type"))) row.append(cell(p.get("total_miners"))) row.append(cell(p.get("miners_hashing"))) row.append(cell(zero, warn=(zero > 10))) # Whole-pod-down explains its own offline (dim it). On an ONLINE # pod, only a CLUSTER of offline (tripped switch/breaker) is a # problem; the 2-3% dead-miner drip is noise. Thresholds TBD. if conn_down: row.append(f"") else: row.append(cell(offline, bad=(offline > 25), warn=(offline > 10))) row.append(cell(sleeping)) row.append(cell(p.get("hashrate"))) row.append(cell(p.get("power"))) row.append(cell(p.get("ems"), warn=bool(p.get("ems")))) row.append(cell(p.get("ems_auto"), warn=bool(p.get("ems_auto")))) row.append(cell(p.get("auto_wake"))) row.append(cell(p.get("auto_reboot"))) row.append("") parts.append("".join(row)) parts.append("
podconntypetotalhashingzeroofflinesleepinghashratepowerEMSEMS_autoAWAR-reboot
{offline}
") parts.append("
") # grp parts.append("
") # site parts.append("") print("".join(parts))