feat: ensemble ML, alert deduplication, larger synthetic dataset

- Replace IsolationForest with voting ensemble (IF + LOF + OC-SVM),
  anomaly flagged when ≥ 2/3 models agree
- Normalise per-process cpu_percent by CPU count to prevent false
  positives from multi-threaded processes (e.g. Firefox 130% → 65%)
- Deduplicate alerts: same PID anomalous within 30s updates last_seen
  and count instead of inserting a new row; WS broadcast only on first
  occurrence
- Show Duration column in Alerts table (e.g. "2м 30с ×28")
- Increase synthetic training data from 2000 to 10000 rows
- Add DB migration for last_seen/count columns on existing databases
This commit is contained in:
kilyabin
2026-05-28 14:25:34 +04:00
parent 20d5e53220
commit 9b7a840ba1
4 changed files with 88 additions and 33 deletions
+1 -1
View File
@@ -153,7 +153,7 @@ class EnsembleDetector:
5 % heavier services (higher RAM, some connections) 5 % heavier services (higher RAM, some connections)
""" """
rng = np.random.default_rng(42) rng = np.random.default_rng(42)
n = 2000 n = 10_000
cpu_idle = rng.exponential(0.3, int(n * 0.75)) cpu_idle = rng.exponential(0.3, int(n * 0.75))
cpu_light = rng.uniform(0.5, 15, int(n * 0.20)) cpu_light = rng.uniform(0.5, 15, int(n * 0.20))
+45 -16
View File
@@ -7,7 +7,7 @@ and pushes real-time anomaly alerts to connected dashboard clients.
import logging import logging
import os import os
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from datetime import date, datetime, timezone from datetime import date, datetime, timedelta, timezone
from typing import Any from typing import Any
import aiosqlite import aiosqlite
@@ -48,7 +48,9 @@ async def _init_db() -> None:
pid INTEGER, pid INTEGER,
process_name TEXT, process_name TEXT,
reason TEXT, reason TEXT,
severity TEXT severity TEXT,
last_seen TEXT,
count INTEGER DEFAULT 1
); );
CREATE TABLE IF NOT EXISTS file_events ( CREATE TABLE IF NOT EXISTS file_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -66,6 +68,14 @@ async def _init_db() -> None:
""") """)
await db.commit() await db.commit()
# Migration for existing databases
for col, definition in [("last_seen", "TEXT"), ("count", "INTEGER DEFAULT 1")]:
try:
await db.execute(f"ALTER TABLE alerts ADD COLUMN {col} {definition}")
await db.commit()
except Exception:
pass
# ── WebSocket manager ───────────────────────────────────────────────────────── # ── WebSocket manager ─────────────────────────────────────────────────────────
@@ -310,23 +320,42 @@ async def post_metrics(batch: MetricsBatch) -> dict[str, int]:
# Все аномалии: ML + пороговые # Все аномалии: ML + пороговые
anomalies = [p for p in batch.processes if p.is_anomaly] anomalies = [p for p in batch.processes if p.is_anomaly]
cutoff = (datetime.now(timezone.utc) - timedelta(seconds=30)).isoformat()
for proc in anomalies: for proc in anomalies:
sev = _severity(proc) sev = _severity(proc)
reason = _reason(proc) reason = _reason(proc)
await db.execute( # Dedup: update existing alert if same process was anomalous recently
"INSERT INTO alerts (timestamp, pid, process_name, reason, severity) VALUES (?,?,?,?,?)", async with db.execute(
(ts, proc.pid, proc.name, reason, sev), """SELECT id FROM alerts
) WHERE pid=? AND process_name=?
await _ws.broadcast( AND (last_seen >= ? OR (last_seen IS NULL AND timestamp >= ?))
{ ORDER BY id DESC LIMIT 1""",
"type": "alert", (proc.pid, proc.name, cutoff, cutoff),
"pid": proc.pid, ) as cur:
"process_name": proc.name, existing = await cur.fetchone()
"reason": reason,
"severity": sev, if existing:
"timestamp": ts, await db.execute(
} "UPDATE alerts SET last_seen=?, count=count+1 WHERE id=?",
) (ts, existing[0]),
)
else:
await db.execute(
"""INSERT INTO alerts
(timestamp, pid, process_name, reason, severity, last_seen, count)
VALUES (?,?,?,?,?,?,1)""",
(ts, proc.pid, proc.name, reason, sev, ts),
)
await _ws.broadcast(
{
"type": "alert",
"pid": proc.pid,
"process_name": proc.name,
"reason": reason,
"severity": sev,
"timestamp": ts,
}
)
await db.commit() await db.commit()
+40 -16
View File
@@ -26,6 +26,16 @@ function severityLabel(s: Alert['severity']): string {
return 'низкий' return 'низкий'
} }
function fmtDuration(first: string, last: string | null): string | null {
if (!last) return null
const secs = Math.round((new Date(last).getTime() - new Date(first).getTime()) / 1000)
if (secs < 5) return null
if (secs < 60) return `${secs}с`
const m = Math.floor(secs / 60)
const s = secs % 60
return s > 0 ? `${m}м ${s}с` : `${m}м`
}
interface Props { interface Props {
wsConnected: boolean wsConnected: boolean
} }
@@ -70,7 +80,8 @@ export function Alerts({ wsConnected }: Props) {
<Table> <Table>
<TableHeader> <TableHeader>
<TableRow> <TableRow>
<TableHead className="w-40">Время</TableHead> <TableHead className="w-40">Начало</TableHead>
<TableHead className="w-28">Длительность</TableHead>
<TableHead className="w-16">PID</TableHead> <TableHead className="w-16">PID</TableHead>
<TableHead>Процесс</TableHead> <TableHead>Процесс</TableHead>
<TableHead>Причина</TableHead> <TableHead>Причина</TableHead>
@@ -78,21 +89,34 @@ export function Alerts({ wsConnected }: Props) {
</TableRow> </TableRow>
</TableHeader> </TableHeader>
<TableBody> <TableBody>
{data.map((a) => ( {data.map((a) => {
<TableRow key={a.id}> const duration = fmtDuration(a.timestamp, a.last_seen)
<TableCell className="font-mono text-xs text-muted-foreground whitespace-nowrap"> return (
{fmtFull(a.timestamp)} <TableRow key={a.id}>
</TableCell> <TableCell className="font-mono text-xs text-muted-foreground whitespace-nowrap">
<TableCell className="font-mono text-muted-foreground">{a.pid}</TableCell> {fmtFull(a.timestamp)}
<TableCell className="font-mono">{a.process_name}</TableCell> </TableCell>
<TableCell className="text-sm text-muted-foreground">{a.reason}</TableCell> <TableCell className="font-mono text-xs whitespace-nowrap">
<TableCell className="text-center"> {duration ? (
<Badge variant={severityVariant(a.severity)}> <span className="text-yellow-400">{duration}</span>
{severityLabel(a.severity)} ) : (
</Badge> <span className="text-muted-foreground"></span>
</TableCell> )}
</TableRow> {(a.count ?? 1) > 1 && (
))} <span className="ml-1 text-muted-foreground">×{a.count}</span>
)}
</TableCell>
<TableCell className="font-mono text-muted-foreground">{a.pid}</TableCell>
<TableCell className="font-mono">{a.process_name}</TableCell>
<TableCell className="text-sm text-muted-foreground">{a.reason}</TableCell>
<TableCell className="text-center">
<Badge variant={severityVariant(a.severity)}>
{severityLabel(a.severity)}
</Badge>
</TableCell>
</TableRow>
)
})}
</TableBody> </TableBody>
</Table> </Table>
)} )}
+2
View File
@@ -17,6 +17,8 @@ export interface ProcessMetric {
export interface Alert { export interface Alert {
id: number id: number
timestamp: string timestamp: string
last_seen: string | null
count: number
pid: number pid: number
process_name: string process_name: string
reason: string reason: string