feat: ensemble ML, alert deduplication, larger synthetic dataset
- Replace IsolationForest with voting ensemble (IF + LOF + OC-SVM), anomaly flagged when ≥ 2/3 models agree - Normalise per-process cpu_percent by CPU count to prevent false positives from multi-threaded processes (e.g. Firefox 130% → 65%) - Deduplicate alerts: same PID anomalous within 30s updates last_seen and count instead of inserting a new row; WS broadcast only on first occurrence - Show Duration column in Alerts table (e.g. "2м 30с ×28") - Increase synthetic training data from 2000 to 10000 rows - Add DB migration for last_seen/count columns on existing databases
This commit is contained in:
+1
-1
@@ -153,7 +153,7 @@ class EnsembleDetector:
|
|||||||
5 % heavier services (higher RAM, some connections)
|
5 % heavier services (higher RAM, some connections)
|
||||||
"""
|
"""
|
||||||
rng = np.random.default_rng(42)
|
rng = np.random.default_rng(42)
|
||||||
n = 2000
|
n = 10_000
|
||||||
|
|
||||||
cpu_idle = rng.exponential(0.3, int(n * 0.75))
|
cpu_idle = rng.exponential(0.3, int(n * 0.75))
|
||||||
cpu_light = rng.uniform(0.5, 15, int(n * 0.20))
|
cpu_light = rng.uniform(0.5, 15, int(n * 0.20))
|
||||||
|
|||||||
+45
-16
@@ -7,7 +7,7 @@ and pushes real-time anomaly alerts to connected dashboard clients.
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
@@ -48,7 +48,9 @@ async def _init_db() -> None:
|
|||||||
pid INTEGER,
|
pid INTEGER,
|
||||||
process_name TEXT,
|
process_name TEXT,
|
||||||
reason TEXT,
|
reason TEXT,
|
||||||
severity TEXT
|
severity TEXT,
|
||||||
|
last_seen TEXT,
|
||||||
|
count INTEGER DEFAULT 1
|
||||||
);
|
);
|
||||||
CREATE TABLE IF NOT EXISTS file_events (
|
CREATE TABLE IF NOT EXISTS file_events (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
@@ -66,6 +68,14 @@ async def _init_db() -> None:
|
|||||||
""")
|
""")
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
|
# Migration for existing databases
|
||||||
|
for col, definition in [("last_seen", "TEXT"), ("count", "INTEGER DEFAULT 1")]:
|
||||||
|
try:
|
||||||
|
await db.execute(f"ALTER TABLE alerts ADD COLUMN {col} {definition}")
|
||||||
|
await db.commit()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
# ── WebSocket manager ─────────────────────────────────────────────────────────
|
# ── WebSocket manager ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -310,23 +320,42 @@ async def post_metrics(batch: MetricsBatch) -> dict[str, int]:
|
|||||||
|
|
||||||
# Все аномалии: ML + пороговые
|
# Все аномалии: ML + пороговые
|
||||||
anomalies = [p for p in batch.processes if p.is_anomaly]
|
anomalies = [p for p in batch.processes if p.is_anomaly]
|
||||||
|
cutoff = (datetime.now(timezone.utc) - timedelta(seconds=30)).isoformat()
|
||||||
for proc in anomalies:
|
for proc in anomalies:
|
||||||
sev = _severity(proc)
|
sev = _severity(proc)
|
||||||
reason = _reason(proc)
|
reason = _reason(proc)
|
||||||
await db.execute(
|
# Dedup: update existing alert if same process was anomalous recently
|
||||||
"INSERT INTO alerts (timestamp, pid, process_name, reason, severity) VALUES (?,?,?,?,?)",
|
async with db.execute(
|
||||||
(ts, proc.pid, proc.name, reason, sev),
|
"""SELECT id FROM alerts
|
||||||
)
|
WHERE pid=? AND process_name=?
|
||||||
await _ws.broadcast(
|
AND (last_seen >= ? OR (last_seen IS NULL AND timestamp >= ?))
|
||||||
{
|
ORDER BY id DESC LIMIT 1""",
|
||||||
"type": "alert",
|
(proc.pid, proc.name, cutoff, cutoff),
|
||||||
"pid": proc.pid,
|
) as cur:
|
||||||
"process_name": proc.name,
|
existing = await cur.fetchone()
|
||||||
"reason": reason,
|
|
||||||
"severity": sev,
|
if existing:
|
||||||
"timestamp": ts,
|
await db.execute(
|
||||||
}
|
"UPDATE alerts SET last_seen=?, count=count+1 WHERE id=?",
|
||||||
)
|
(ts, existing[0]),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await db.execute(
|
||||||
|
"""INSERT INTO alerts
|
||||||
|
(timestamp, pid, process_name, reason, severity, last_seen, count)
|
||||||
|
VALUES (?,?,?,?,?,?,1)""",
|
||||||
|
(ts, proc.pid, proc.name, reason, sev, ts),
|
||||||
|
)
|
||||||
|
await _ws.broadcast(
|
||||||
|
{
|
||||||
|
"type": "alert",
|
||||||
|
"pid": proc.pid,
|
||||||
|
"process_name": proc.name,
|
||||||
|
"reason": reason,
|
||||||
|
"severity": sev,
|
||||||
|
"timestamp": ts,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,16 @@ function severityLabel(s: Alert['severity']): string {
|
|||||||
return 'низкий'
|
return 'низкий'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function fmtDuration(first: string, last: string | null): string | null {
|
||||||
|
if (!last) return null
|
||||||
|
const secs = Math.round((new Date(last).getTime() - new Date(first).getTime()) / 1000)
|
||||||
|
if (secs < 5) return null
|
||||||
|
if (secs < 60) return `${secs}с`
|
||||||
|
const m = Math.floor(secs / 60)
|
||||||
|
const s = secs % 60
|
||||||
|
return s > 0 ? `${m}м ${s}с` : `${m}м`
|
||||||
|
}
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
wsConnected: boolean
|
wsConnected: boolean
|
||||||
}
|
}
|
||||||
@@ -70,7 +80,8 @@ export function Alerts({ wsConnected }: Props) {
|
|||||||
<Table>
|
<Table>
|
||||||
<TableHeader>
|
<TableHeader>
|
||||||
<TableRow>
|
<TableRow>
|
||||||
<TableHead className="w-40">Время</TableHead>
|
<TableHead className="w-40">Начало</TableHead>
|
||||||
|
<TableHead className="w-28">Длительность</TableHead>
|
||||||
<TableHead className="w-16">PID</TableHead>
|
<TableHead className="w-16">PID</TableHead>
|
||||||
<TableHead>Процесс</TableHead>
|
<TableHead>Процесс</TableHead>
|
||||||
<TableHead>Причина</TableHead>
|
<TableHead>Причина</TableHead>
|
||||||
@@ -78,21 +89,34 @@ export function Alerts({ wsConnected }: Props) {
|
|||||||
</TableRow>
|
</TableRow>
|
||||||
</TableHeader>
|
</TableHeader>
|
||||||
<TableBody>
|
<TableBody>
|
||||||
{data.map((a) => (
|
{data.map((a) => {
|
||||||
<TableRow key={a.id}>
|
const duration = fmtDuration(a.timestamp, a.last_seen)
|
||||||
<TableCell className="font-mono text-xs text-muted-foreground whitespace-nowrap">
|
return (
|
||||||
{fmtFull(a.timestamp)}
|
<TableRow key={a.id}>
|
||||||
</TableCell>
|
<TableCell className="font-mono text-xs text-muted-foreground whitespace-nowrap">
|
||||||
<TableCell className="font-mono text-muted-foreground">{a.pid}</TableCell>
|
{fmtFull(a.timestamp)}
|
||||||
<TableCell className="font-mono">{a.process_name}</TableCell>
|
</TableCell>
|
||||||
<TableCell className="text-sm text-muted-foreground">{a.reason}</TableCell>
|
<TableCell className="font-mono text-xs whitespace-nowrap">
|
||||||
<TableCell className="text-center">
|
{duration ? (
|
||||||
<Badge variant={severityVariant(a.severity)}>
|
<span className="text-yellow-400">{duration}</span>
|
||||||
{severityLabel(a.severity)}
|
) : (
|
||||||
</Badge>
|
<span className="text-muted-foreground">—</span>
|
||||||
</TableCell>
|
)}
|
||||||
</TableRow>
|
{(a.count ?? 1) > 1 && (
|
||||||
))}
|
<span className="ml-1 text-muted-foreground">×{a.count}</span>
|
||||||
|
)}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell className="font-mono text-muted-foreground">{a.pid}</TableCell>
|
||||||
|
<TableCell className="font-mono">{a.process_name}</TableCell>
|
||||||
|
<TableCell className="text-sm text-muted-foreground">{a.reason}</TableCell>
|
||||||
|
<TableCell className="text-center">
|
||||||
|
<Badge variant={severityVariant(a.severity)}>
|
||||||
|
{severityLabel(a.severity)}
|
||||||
|
</Badge>
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
)
|
||||||
|
})}
|
||||||
</TableBody>
|
</TableBody>
|
||||||
</Table>
|
</Table>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ export interface ProcessMetric {
|
|||||||
export interface Alert {
|
export interface Alert {
|
||||||
id: number
|
id: number
|
||||||
timestamp: string
|
timestamp: string
|
||||||
|
last_seen: string | null
|
||||||
|
count: number
|
||||||
pid: number
|
pid: number
|
||||||
process_name: string
|
process_name: string
|
||||||
reason: string
|
reason: string
|
||||||
|
|||||||
Reference in New Issue
Block a user