feat: ensemble ML, alert deduplication, larger synthetic dataset

- Replace IsolationForest with voting ensemble (IF + LOF + OC-SVM),
  anomaly flagged when ≥ 2/3 models agree
- Normalise per-process cpu_percent by CPU count to prevent false
  positives from multi-threaded processes (e.g. Firefox 130% → 65%)
- Deduplicate alerts: same PID anomalous within 30s updates last_seen
  and count instead of inserting a new row; WS broadcast only on first
  occurrence
- Show Duration column in Alerts table (e.g. "2м 30с ×28")
- Increase synthetic training data from 2000 to 10000 rows
- Add DB migration for last_seen/count columns on existing databases
This commit is contained in:
kilyabin
2026-05-28 14:25:34 +04:00
parent 20d5e53220
commit 9b7a840ba1
4 changed files with 88 additions and 33 deletions
+1 -1
View File
@@ -153,7 +153,7 @@ class EnsembleDetector:
5 % heavier services (higher RAM, some connections)
"""
rng = np.random.default_rng(42)
n = 2000
n = 10_000
cpu_idle = rng.exponential(0.3, int(n * 0.75))
cpu_light = rng.uniform(0.5, 15, int(n * 0.20))
+33 -4
View File
@@ -7,7 +7,7 @@ and pushes real-time anomaly alerts to connected dashboard clients.
import logging
import os
from contextlib import asynccontextmanager
from datetime import date, datetime, timezone
from datetime import date, datetime, timedelta, timezone
from typing import Any
import aiosqlite
@@ -48,7 +48,9 @@ async def _init_db() -> None:
pid INTEGER,
process_name TEXT,
reason TEXT,
severity TEXT
severity TEXT,
last_seen TEXT,
count INTEGER DEFAULT 1
);
CREATE TABLE IF NOT EXISTS file_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -66,6 +68,14 @@ async def _init_db() -> None:
""")
await db.commit()
# Migration for existing databases
for col, definition in [("last_seen", "TEXT"), ("count", "INTEGER DEFAULT 1")]:
try:
await db.execute(f"ALTER TABLE alerts ADD COLUMN {col} {definition}")
await db.commit()
except Exception:
pass
# ── WebSocket manager ─────────────────────────────────────────────────────────
@@ -310,12 +320,31 @@ async def post_metrics(batch: MetricsBatch) -> dict[str, int]:
# Все аномалии: ML + пороговые
anomalies = [p for p in batch.processes if p.is_anomaly]
cutoff = (datetime.now(timezone.utc) - timedelta(seconds=30)).isoformat()
for proc in anomalies:
sev = _severity(proc)
reason = _reason(proc)
# Dedup: update existing alert if same process was anomalous recently
async with db.execute(
"""SELECT id FROM alerts
WHERE pid=? AND process_name=?
AND (last_seen >= ? OR (last_seen IS NULL AND timestamp >= ?))
ORDER BY id DESC LIMIT 1""",
(proc.pid, proc.name, cutoff, cutoff),
) as cur:
existing = await cur.fetchone()
if existing:
await db.execute(
"INSERT INTO alerts (timestamp, pid, process_name, reason, severity) VALUES (?,?,?,?,?)",
(ts, proc.pid, proc.name, reason, sev),
"UPDATE alerts SET last_seen=?, count=count+1 WHERE id=?",
(ts, existing[0]),
)
else:
await db.execute(
"""INSERT INTO alerts
(timestamp, pid, process_name, reason, severity, last_seen, count)
VALUES (?,?,?,?,?,?,1)""",
(ts, proc.pid, proc.name, reason, sev, ts),
)
await _ws.broadcast(
{
+27 -3
View File
@@ -26,6 +26,16 @@ function severityLabel(s: Alert['severity']): string {
return 'низкий'
}
function fmtDuration(first: string, last: string | null): string | null {
if (!last) return null
const secs = Math.round((new Date(last).getTime() - new Date(first).getTime()) / 1000)
if (secs < 5) return null
if (secs < 60) return `${secs}с`
const m = Math.floor(secs / 60)
const s = secs % 60
return s > 0 ? `${m}м ${s}с` : `${m}м`
}
interface Props {
wsConnected: boolean
}
@@ -70,7 +80,8 @@ export function Alerts({ wsConnected }: Props) {
<Table>
<TableHeader>
<TableRow>
<TableHead className="w-40">Время</TableHead>
<TableHead className="w-40">Начало</TableHead>
<TableHead className="w-28">Длительность</TableHead>
<TableHead className="w-16">PID</TableHead>
<TableHead>Процесс</TableHead>
<TableHead>Причина</TableHead>
@@ -78,11 +89,23 @@ export function Alerts({ wsConnected }: Props) {
</TableRow>
</TableHeader>
<TableBody>
{data.map((a) => (
{data.map((a) => {
const duration = fmtDuration(a.timestamp, a.last_seen)
return (
<TableRow key={a.id}>
<TableCell className="font-mono text-xs text-muted-foreground whitespace-nowrap">
{fmtFull(a.timestamp)}
</TableCell>
<TableCell className="font-mono text-xs whitespace-nowrap">
{duration ? (
<span className="text-yellow-400">{duration}</span>
) : (
<span className="text-muted-foreground"></span>
)}
{(a.count ?? 1) > 1 && (
<span className="ml-1 text-muted-foreground">×{a.count}</span>
)}
</TableCell>
<TableCell className="font-mono text-muted-foreground">{a.pid}</TableCell>
<TableCell className="font-mono">{a.process_name}</TableCell>
<TableCell className="text-sm text-muted-foreground">{a.reason}</TableCell>
@@ -92,7 +115,8 @@ export function Alerts({ wsConnected }: Props) {
</Badge>
</TableCell>
</TableRow>
))}
)
})}
</TableBody>
</Table>
)}
+2
View File
@@ -17,6 +17,8 @@ export interface ProcessMetric {
export interface Alert {
id: number
timestamp: string
last_seen: string | null
count: number
pid: number
process_name: string
reason: string