#!/usr/bin/env python3
import cgi, sys
sys.path.insert(0, '/opt/ngon/apps')
from managers.auth_manager import AuthManager, generate_login_page_html
_form = cgi.FieldStorage()
_auth = AuthManager('dead')
_auth_required, _should_exit, _headers = _auth.require_auth(_form)
if _should_exit:
print("Content-Type: application/json")
if _headers:
print(_headers)
print("")
if _auth_required:
print('{"success": false, "error": "Authentication required"}')
else:
print('{"success": true}')
sys.exit(0)
if _auth_required:
print("Content-Type: text/html")
print("")
print(generate_login_page_html("Dead Miner Hunter"))
sys.exit(0)
"""
Dead Miner Hunter - Redesigned Interface
File: /var/www/html/ngon/nmm/dead.py
Identifies miners with errors and presents them in a filterable table view.
"""
import pandas as pd
import json
import os
import sys
from html import escape
from datetime import datetime, timedelta
sys.path.insert(0, '/opt/ngon/apps')
from managers.config_manager import config
from miners.error_categories import category_for
# Scoring logic + the DeadMinerHunter class live in the shared backend module
# so this page, the signal-extraction cron and the inventory-facing API all
# score from ONE implementation with ONE config. Tunables come from
# master_config -> misc.dead_miner (Site Manager panel); see dead_hunter.py.
from miners.dead_hunter import ( # noqa: E402
DeadMinerHunter,
load_config as load_dead_config,
score_signals,
score_store,
)
_DEAD_CFG = load_dead_config()
DEAD_PULL_THRESHOLD = _DEAD_CFG['pull_threshold']
DEAD_SCORE_FLOOR = _DEAD_CFG['score_floor']
def build_miners_json(hunter, miners):
"""Build JSON data for miners (with errors, or otherwise notable)."""
miners_json = []
for miner in miners:
raw_mac = str(miner.get('mac', ''))
mac = raw_mac.strip().upper() if raw_mac else ''
serial = str(miner.get('serial', 'Unknown'))
last5 = serial[-5:] if len(serial) >= 5 else serial
cd = hunter.confirmed_dead.get(mac, {}) if mac else {}
has_hashboard_err = any(category_for(e) == 'hashboard' for e in miner.get('errors', []))
miners_json.append({
'mac': mac,
'ip': str(miner.get('ip', '')),
'pod': miner.get('pod', 'Unknown'),
'position': str(miner.get('position', 'Unknown')),
'serial': serial,
'serial_short': last5,
'site': str(miner.get('site', 'Unknown')),
'hashrate': miner.get('hashrate', 0),
'errors': miner.get('errors', []),
'reboots': hunter.get_reboot_count(raw_mac),
'reboots_life': hunter.get_lifetime_reboot_count(raw_mac),
'dead_score': cd.get('score', 0),
'dead_needs_setup': cd.get('needs_setup', False),
'dead_detail': cd,
'has_hashboard_error': has_hashboard_err,
'is_sleeping': bool(mac and mac in hunter.sleeping_miners),
})
return miners_json
def main():
# AJAX entry point: lazy-load dead-confidence scores when the user clicks
# the pill. compute_confirmed_dead loads its own 30d reboot data, so the
# full reboot InfluxDB pull is skipped here.
action = _form.getvalue('action') if '_form' in globals() else None
if action == 'confirmed_dead':
# Scores come from the persisted signal store, scored with the weights
# configured right now — milliseconds, versus the ~90 s live history
# scan this used to do. The cron (dead_signals_job.py) owns the
# expensive extraction. If the store is missing (first deploy, or a
# failed cron), fall back to computing live so the page still works.
records, extracted_at = score_store()
used_store = bool(records) or extracted_at is not None
if not used_store:
hunter = DeadMinerHunter(skip_reboots=True)
hunter.compute_confirmed_dead()
if not hunter.lifetime_reboot_counts and hunter.influx_query_api:
hunter._query_reboots('0', hunter.lifetime_reboot_counts)
records = []
for mac, info in hunter.confirmed_dead.items():
sig = hunter.signals.get(mac, {})
records.append({
'mac': mac, 'pod': sig.get('pod', 'Unknown'),
'site': sig.get('site', 'Unknown'),
'position': sig.get('position', 'Unknown'),
'serial': sig.get('serial', 'Unknown'),
'hashrate': sig.get('hs_rt'),
'mining_state': sig.get('mining_state'),
'offline': sig.get('offline', False),
'errors': sig.get('errors', []),
'reboots_life': hunter.get_lifetime_reboot_count(mac),
'score': info.get('score', 0),
'tier': info.get('tier', 'low'),
'signals': {'reboots_30d': info.get('reboots_30d', 0),
'last_hash': info.get('last_hash')},
'detail': info,
})
# Live IP + sleep state aren't in the store (they change every ~30 s);
# join them from the current scan so the table's IP column and the blue
# sleeping-row tint still work.
ip_by_mac, sleeping = {}, set()
try:
df_live = pd.read_csv('/opt/ngon/data/live/miner_status.csv')
for r in df_live.itertuples(index=False):
m = str(getattr(r, 'mac', '') or '').strip().upper()
if not m:
continue
ip_by_mac[m] = str(getattr(r, 'ip', '') or '')
if str(getattr(r, 'mining_state', '') or '').lower() == 'sleeping':
sleeping.add(m)
except Exception:
pass
# Reshape the store's records into the field names the page's JS uses.
out = []
for rec in records:
mac = rec['mac']
info = rec.get('detail', {})
serial = str(rec.get('serial') or 'Unknown')
errs = rec.get('errors', []) or []
out.append({
'mac': mac,
'ip': ip_by_mac.get(mac, ''),
'pod': rec.get('pod') or 'Unknown',
'position': str(rec.get('position') or 'Unknown'),
'serial': serial,
'serial_short': serial[-5:] if len(serial) >= 5 else serial,
'site': str(rec.get('site') or 'Unknown'),
'hashrate': rec.get('hashrate'),
'errors': errs,
'reboots': rec.get('signals', {}).get('reboots_30d', 0),
'reboots_life': rec.get('reboots_life', 0),
'offline': rec.get('offline', False),
'has_hashboard_error': any(category_for(e) == 'hashboard' for e in errs),
'is_sleeping': mac in sleeping,
'dead_score': rec.get('score', 0),
'dead_needs_setup': info.get('needs_setup', False),
'last_hash': rec.get('signals', {}).get('last_hash'),
'dead_detail': info,
})
# Pill count is pull candidates only; records carry the full
# amber-and-up set so their gauges render too.
pull_count = sum(1 for r in out if r['dead_score'] >= DEAD_PULL_THRESHOLD)
print("Content-Type: application/json\n")
print(json.dumps({'dead_miners': out, 'count': pull_count,
'extracted_at': extracted_at,
'source': 'store' if used_store else 'live'}))
return
hunter = DeadMinerHunter()
active_miners = hunter.get_active_miners()
# Confirmed-dead is not pre-computed on page load; the table only shows
# miners-with-errors (active_miners) + offline. Confirmed Dead miners that
# would otherwise be invisible (zero-hash-no-errors, sleeping) get merged in
# client-side after the user clicks the Confirmed Dead pill.
table_miners = active_miners
# Build error info
all_errors = set()
for miner in active_miners:
all_errors.update(miner['errors'])
error_info = []
for error_code in sorted(list(all_errors)):
info = hunter.get_error_info(error_code)
miner_count = sum(1 for miner in active_miners if error_code in miner['errors'])
error_info.append({
'code': error_code,
'meaning': info['meaning'],
'solution': info['solution'],
'count': miner_count,
'category': category_for(error_code) or 'uncategorized',
})
# Build pod list with counts (use union: active errors + extra confirmed-dead)
pod_counts = {}
for miner in table_miners:
pod = miner.get('pod', 'Unknown')
pod_counts[pod] = pod_counts.get(pod, 0) + 1
# Build pod ordering from master_config (site order, then pod order within each site)
pod_order = []
try:
import json as _json
with open('/opt/ngon/config/master_config.json') as f:
cfg = _json.load(f)
for site_name, site_data in cfg.get('sites', {}).items():
for group_name, group_data in site_data.get('generator_groups', {}).items():
for pod_name in group_data.get('pods', {}):
pod_order.append(pod_name)
except Exception:
pod_order = sorted(pod_counts.keys())
# All error codes from CSV for keyword matching
all_error_details = {code: info for code, info in hunter.error_codes.items()}
for e in error_info:
if e['code'] not in all_error_details:
all_error_details[e['code']] = {'meaning': e['meaning'], 'solution': e['solution']}
miners_json = build_miners_json(hunter, table_miners)
zero_hash_with_errors = hunter.get_zero_hash_with_errors_count()
zero_hash_no_errors = hunter.get_zero_hash_no_errors_count()
# Synthesize offline-miner records and merge into the same JSON the table consumes.
offline_json = []
for o in hunter.offline_miners:
serial = o.get('serial') or ''
last5 = serial[-5:] if len(serial) >= 5 else serial
mac_u = (o.get('mac', '') or '').upper()
cd = hunter.confirmed_dead.get(mac_u, {})
offline_json.append({
'mac': o.get('mac', ''),
'ip': '',
'pod': o.get('pod', 'Unknown'),
'position': o.get('position', ''),
'serial': serial or 'Unknown',
'serial_short': last5 or '',
'site': o.get('site', 'Unknown'),
'hashrate': None,
'errors': [],
'reboots': hunter.get_reboot_count(o.get('mac', '')),
'reboots_life': hunter.get_lifetime_reboot_count(o.get('mac', '')),
'offline': True,
'dead_score': cd.get('score', 0),
'dead_needs_setup': cd.get('needs_setup', False),
'dead_detail': cd,
'has_hashboard_error': False,
'is_sleeping': False,
})
# Nav imports
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from links import generate_dropdown_css, generate_dropdown_js, generate_dropdown_html
print("Content-Type: text/html\n")
print(f"""
Dead Miner Hunter