/** * Security Intelligence & Pattern Analysis * Algorithm-driven surveillance system for automatic pattern detection * Includes anomaly detection, threat intelligence, and predictive analysis * Enhanced with configurable thresholds and risk signatures (CWE-778) */ const logger = require('./logger'); const logAggregator = require('./logAggregator'); const { db } = require('../database/db'); const thresholdManager = require('./thresholdManager'); const riskSignatureManager = require('./riskSignatureManager'); class SecurityIntelligence { constructor() { this.patterns = new Map(); this.anomalies = []; this.threatScore = 0; this.analysisInterval = 60000; // 1 minute this.initialize(); } /** * Initialize security intelligence system */ async initialize() { await this.createAnomaliesTable(); await this.createThreatIntelligenceTable(); // Start continuous monitoring setInterval(() => this.analyze(), this.analysisInterval); logger.info('[SecurityIntelligence] Initialized - Active monitoring enabled'); } /** * Create anomalies table */ async createAnomaliesTable() { return new Promise((resolve, reject) => { db.run(` CREATE TABLE IF NOT EXISTS security_anomalies ( id INTEGER PRIMARY KEY AUTOINCREMENT, anomaly_id TEXT UNIQUE NOT NULL, type TEXT NOT NULL, severity TEXT NOT NULL, description TEXT NOT NULL, confidence REAL NOT NULL, affected_user_id INTEGER, affected_ip TEXT, pattern_data TEXT, related_logs TEXT, status TEXT DEFAULT 'open', resolved_at DATETIME, resolved_by INTEGER, resolution_notes TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) `, (err) => { if (err) reject(err); else { db.run(`CREATE INDEX IF NOT EXISTS idx_anomalies_type ON security_anomalies(type, created_at DESC)`); db.run(`CREATE INDEX IF NOT EXISTS idx_anomalies_severity ON security_anomalies(severity, created_at DESC)`); db.run(`CREATE INDEX IF NOT EXISTS idx_anomalies_status ON security_anomalies(status, created_at DESC)`); resolve(); } }); }); } /** * Create threat intelligence table */ async createThreatIntelligenceTable() { return new Promise((resolve, reject) => { db.run(` CREATE TABLE IF NOT EXISTS threat_intelligence ( id INTEGER PRIMARY KEY AUTOINCREMENT, indicator TEXT NOT NULL, indicator_type TEXT NOT NULL, threat_level TEXT NOT NULL, description TEXT, source TEXT, confidence REAL NOT NULL, first_seen DATETIME DEFAULT CURRENT_TIMESTAMP, last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, occurrence_count INTEGER DEFAULT 1, metadata TEXT ) `, (err) => { if (err) reject(err); else { db.run(`CREATE UNIQUE INDEX IF NOT EXISTS idx_threat_indicator ON threat_intelligence(indicator, indicator_type)`); db.run(`CREATE INDEX IF NOT EXISTS idx_threat_level ON threat_intelligence(threat_level, last_seen DESC)`); resolve(); } }); }); } /** * Main analysis loop - runs continuously */ async analyze() { try { logger.debug('[SecurityIntelligence] Running analysis cycle'); // Run all detection algorithms in parallel await Promise.all([ this.detectBruteForceAttacks(), this.detectAccountEnumeration(), this.detectPrivilegeEscalation(), this.detectAnomalousAccess(), this.detectSuspiciousIPs(), this.detectDataExfiltration(), this.detectSessionAnomalies(), this.detectRateLimitAbuse() ]); // Calculate overall threat score await this.calculateThreatScore(); } catch (error) { logger.error('[SecurityIntelligence] Analysis cycle failed:', error); } } /** * Detect brute force authentication attacks * Enhanced with configurable thresholds */ async detectBruteForceAttacks() { // Get configured threshold or use default const thresholdConfig = await thresholdManager.getThresholds({ patternType: 'brute_force_attack' }); const timeWindow = thresholdConfig[0]?.time_window_minutes || 10; const threshold = thresholdConfig[0]?.threshold_value || 10; const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT ip_address, COUNT(*) as attempt_count, MAX(timestamp) as last_attempt, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category = 'authentication' AND level IN ('warn', 'error') AND message LIKE '%failed%' AND timestamp >= ? AND ip_address IS NOT NULL GROUP BY ip_address HAVING attempt_count >= ?`, [startTime, threshold], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { // Evaluate threshold const thresholdResult = await thresholdManager.evaluateThreshold( 'brute_force_attack', 'failed_login_count', row.attempt_count, { ip_address: row.ip_address, timeWindow } ); if (!thresholdResult.exceeded) continue; const configuredThreshold = thresholdResult.thresholds[0]; const severity = configuredThreshold?.severity || (row.attempt_count > 20 ? 'critical' : 'high'); await this.createAnomaly({ type: 'brute_force_attack', severity: severity, description: `Brute force attack detected from IP ${row.ip_address}: ${row.attempt_count} failed login attempts in ${timeWindow} minutes (threshold: ${threshold})`, confidence: Math.min(row.attempt_count / threshold, 1.0), affected_ip: row.ip_address, pattern_data: JSON.stringify({ attemptCount: row.attempt_count, timeWindow: `${timeWindow} minutes`, threshold: threshold, thresholdExceeded: thresholdResult.exceeded, lastAttempt: row.last_attempt }), related_logs: row.log_ids }); // Add to threat intelligence await this.addThreatIndicator(row.ip_address, 'ip', severity === 'critical' ? 'critical' : 'high', 'Brute force attack source'); } resolve(rows.length); } ); }); } /** * Detect account enumeration attempts * Enhanced with configurable thresholds */ async detectAccountEnumeration() { // Get configured threshold or use default const thresholdConfig = await thresholdManager.getThresholds({ patternType: 'credential_stuffing' }); const timeWindow = thresholdConfig[0]?.time_window_minutes || 5; const threshold = thresholdConfig[0]?.threshold_value || 5; const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT ip_address, COUNT(DISTINCT json_extract(metadata, '$.username')) as unique_usernames, COUNT(*) as total_attempts, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category = 'authentication' AND level = 'warn' AND timestamp >= ? AND ip_address IS NOT NULL AND metadata LIKE '%username%' GROUP BY ip_address HAVING unique_usernames >= ?`, [startTime, threshold], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'account_enumeration', severity: 'medium', description: `Account enumeration detected from IP ${row.ip_address}: ${row.unique_usernames} different usernames tried in ${timeWindow} minutes`, confidence: Math.min(row.unique_usernames / (threshold * 2), 1.0), affected_ip: row.ip_address, pattern_data: JSON.stringify({ uniqueUsernames: row.unique_usernames, totalAttempts: row.total_attempts, timeWindow: `${timeWindow} minutes` }), related_logs: row.log_ids }); } resolve(rows.length); } ); }); } /** * Detect privilege escalation attempts */ async detectPrivilegeEscalation() { const timeWindow = 30; // minutes const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT user_id, ip_address, COUNT(*) as escalation_attempts, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category = 'authorization' AND (message LIKE '%denied%' OR message LIKE '%unauthorized%') AND timestamp >= ? AND user_id IS NOT NULL GROUP BY user_id, ip_address HAVING escalation_attempts >= 3`, [startTime], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'privilege_escalation', severity: 'critical', description: `Privilege escalation attempt detected: User ${row.user_id} attempted ${row.escalation_attempts} unauthorized actions`, confidence: 0.85, affected_user_id: row.user_id, affected_ip: row.ip_address, pattern_data: JSON.stringify({ escalationAttempts: row.escalation_attempts, timeWindow: `${timeWindow} minutes` }), related_logs: row.log_ids }); } resolve(rows.length); } ); }); } /** * Detect anomalous access patterns */ async detectAnomalousAccess() { const timeWindow = 60; // minutes const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); // Detect access from unusual hours (2 AM - 5 AM) return new Promise((resolve, reject) => { db.all( `SELECT user_id, ip_address, COUNT(*) as access_count, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category IN ('access', 'security_audit') AND timestamp >= ? AND CAST(strftime('%H', timestamp) AS INTEGER) BETWEEN 2 AND 5 AND user_id IS NOT NULL GROUP BY user_id, ip_address HAVING access_count >= 3`, [startTime], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'anomalous_access', severity: 'medium', description: `Unusual access pattern: User ${row.user_id} accessed system during off-hours (${row.access_count} times)`, confidence: 0.7, affected_user_id: row.user_id, affected_ip: row.ip_address, pattern_data: JSON.stringify({ accessCount: row.access_count, timeRange: '2 AM - 5 AM' }), related_logs: row.log_ids }); } resolve(rows.length); } ); }); } /** * Detect suspicious IP addresses */ async detectSuspiciousIPs() { const timeWindow = 60; // minutes const threshold = 100; // requests const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT ip_address, COUNT(*) as request_count, COUNT(DISTINCT user_id) as unique_users, COUNT(CASE WHEN level = 'error' THEN 1 END) as error_count, GROUP_CONCAT(DISTINCT source) as sources FROM aggregated_logs WHERE timestamp >= ? AND ip_address IS NOT NULL GROUP BY ip_address HAVING request_count >= ? OR unique_users >= 10`, [startTime, threshold], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { let severity = 'low'; let reason = []; if (row.request_count >= threshold * 2) { severity = 'high'; reason.push(`excessive requests (${row.request_count})`); } else if (row.request_count >= threshold) { severity = 'medium'; reason.push(`high request volume (${row.request_count})`); } if (row.unique_users >= 10) { severity = 'high'; reason.push(`multiple user accounts (${row.unique_users})`); } if (row.error_count > row.request_count * 0.3) { severity = 'high'; reason.push(`high error rate (${row.error_count})`); } await this.createAnomaly({ type: 'suspicious_ip', severity, description: `Suspicious IP activity from ${row.ip_address}: ${reason.join(', ')}`, confidence: 0.75, affected_ip: row.ip_address, pattern_data: JSON.stringify({ requestCount: row.request_count, uniqueUsers: row.unique_users, errorCount: row.error_count, sources: row.sources }), related_logs: null }); if (severity === 'high') { await this.addThreatIndicator(row.ip_address, 'ip', severity, reason.join(', ')); } } resolve(rows.length); } ); }); } /** * Detect potential data exfiltration */ async detectDataExfiltration() { const timeWindow = 30; // minutes const downloadThreshold = 5; // Large downloads const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT user_id, ip_address, COUNT(*) as download_count, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category = 'access' AND (message LIKE '%download%' OR message LIKE '%export%' OR message LIKE '%backup%') AND timestamp >= ? AND user_id IS NOT NULL GROUP BY user_id, ip_address HAVING download_count >= ?`, [startTime, downloadThreshold], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'data_exfiltration', severity: 'high', description: `Potential data exfiltration: User ${row.user_id} performed ${row.download_count} download/export operations in ${timeWindow} minutes`, confidence: 0.8, affected_user_id: row.user_id, affected_ip: row.ip_address, pattern_data: JSON.stringify({ downloadCount: row.download_count, timeWindow: `${timeWindow} minutes` }), related_logs: row.log_ids }); } resolve(rows.length); } ); }); } /** * Detect session anomalies */ async detectSessionAnomalies() { const timeWindow = 24; // hours const startTime = new Date(Date.now() - timeWindow * 60 * 60 * 1000).toISOString(); // Detect impossible travel (same user, different locations in short time) return new Promise((resolve, reject) => { db.all( `SELECT user_id, COUNT(DISTINCT ip_address) as unique_ips, GROUP_CONCAT(DISTINCT ip_address) as ips, COUNT(*) as session_count FROM aggregated_logs WHERE category = 'authentication' AND message LIKE '%login%success%' AND timestamp >= ? AND user_id IS NOT NULL GROUP BY user_id HAVING unique_ips >= 5`, [startTime], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'session_anomaly', severity: 'medium', description: `Session anomaly: User ${row.user_id} logged in from ${row.unique_ips} different IP addresses in ${timeWindow} hours`, confidence: 0.7, affected_user_id: row.user_id, pattern_data: JSON.stringify({ uniqueIPs: row.unique_ips, ipAddresses: row.ips.split(','), sessionCount: row.session_count, timeWindow: `${timeWindow} hours` }), related_logs: null }); } resolve(rows.length); } ); }); } /** * Detect rate limit abuse */ async detectRateLimitAbuse() { const timeWindow = 15; // minutes const startTime = new Date(Date.now() - timeWindow * 60 * 1000).toISOString(); return new Promise((resolve, reject) => { db.all( `SELECT ip_address, COUNT(*) as blocked_count, GROUP_CONCAT(log_id) as log_ids FROM aggregated_logs WHERE category = 'system' AND message LIKE '%rate limit%' AND timestamp >= ? AND ip_address IS NOT NULL GROUP BY ip_address HAVING blocked_count >= 5`, [startTime], async (err, rows) => { if (err) { reject(err); return; } for (const row of rows) { await this.createAnomaly({ type: 'rate_limit_abuse', severity: 'medium', description: `Rate limit abuse: IP ${row.ip_address} was rate-limited ${row.blocked_count} times in ${timeWindow} minutes`, confidence: 0.9, affected_ip: row.ip_address, pattern_data: JSON.stringify({ blockedCount: row.blocked_count, timeWindow: `${timeWindow} minutes` }), related_logs: row.log_ids }); await this.addThreatIndicator(row.ip_address, 'ip', 'medium', 'Rate limit abuse'); } resolve(rows.length); } ); }); } /** * Create anomaly record */ async createAnomaly(details) { const anomalyId = `ANOM-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; // Check if similar anomaly exists (deduplication) const existing = await this.findSimilarAnomaly(details); if (existing) { logger.debug(`[SecurityIntelligence] Similar anomaly exists: ${existing.anomaly_id}`); return existing.anomaly_id; } return new Promise((resolve, reject) => { db.run( `INSERT INTO security_anomalies (anomaly_id, type, severity, description, confidence, affected_user_id, affected_ip, pattern_data, related_logs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ anomalyId, details.type, details.severity, details.description, details.confidence, details.affected_user_id || null, details.affected_ip || null, details.pattern_data, details.related_logs ], (err) => { if (err) { reject(err); } else { logger.warn(`[SecurityIntelligence] Anomaly detected: ${details.type} - ${details.severity} - ${details.description}`); // Log to aggregated logs as well logAggregator.aggregate('security_intelligence', 'warn', 'security', details.description, { anomalyId, type: details.type, severity: details.severity, confidence: details.confidence }); resolve(anomalyId); } } ); }); } /** * Find similar anomaly (deduplication) */ async findSimilarAnomaly(details) { const recentTime = new Date(Date.now() - 10 * 60 * 1000).toISOString(); // Last 10 minutes return new Promise((resolve, reject) => { db.get( `SELECT * FROM security_anomalies WHERE type = ? AND severity = ? AND (affected_user_id = ? OR affected_ip = ?) AND status = 'open' AND created_at >= ? ORDER BY created_at DESC LIMIT 1`, [details.type, details.severity, details.affected_user_id, details.affected_ip, recentTime], (err, row) => { if (err) reject(err); else resolve(row); } ); }); } /** * Add threat indicator to intelligence database */ async addThreatIndicator(indicator, type, level, description) { return new Promise((resolve, reject) => { db.run( `INSERT INTO threat_intelligence (indicator, indicator_type, threat_level, description, confidence, source) VALUES (?, ?, ?, ?, 0.8, 'internal_detection') ON CONFLICT(indicator, indicator_type) DO UPDATE SET last_seen = CURRENT_TIMESTAMP, occurrence_count = occurrence_count + 1, threat_level = ?, description = ?`, [indicator, type, level, description, level, description], (err) => { if (err) reject(err); else { logger.info(`[SecurityIntelligence] Threat indicator added: ${type}=${indicator} (${level})`); resolve(); } } ); }); } /** * Calculate overall threat score (0-100) */ async calculateThreatScore() { return new Promise((resolve, reject) => { db.get( `SELECT COUNT(CASE WHEN severity = 'critical' AND status = 'open' THEN 1 END) as critical_count, COUNT(CASE WHEN severity = 'high' AND status = 'open' THEN 1 END) as high_count, COUNT(CASE WHEN severity = 'medium' AND status = 'open' THEN 1 END) as medium_count, COUNT(CASE WHEN severity = 'low' AND status = 'open' THEN 1 END) as low_count FROM security_anomalies WHERE created_at >= datetime('now', '-24 hours')`, [], (err, row) => { if (err) { reject(err); return; } // Weight severity levels const score = Math.min( (row.critical_count * 40) + (row.high_count * 20) + (row.medium_count * 10) + (row.low_count * 5), 100 ); this.threatScore = score; if (score >= 80) { logger.error(`[SecurityIntelligence] CRITICAL THREAT LEVEL: ${score}/100`); } else if (score >= 50) { logger.warn(`[SecurityIntelligence] HIGH THREAT LEVEL: ${score}/100`); } else if (score >= 20) { logger.info(`[SecurityIntelligence] MEDIUM THREAT LEVEL: ${score}/100`); } else { logger.debug(`[SecurityIntelligence] LOW THREAT LEVEL: ${score}/100`); } resolve(score); } ); }); } /** * Get active anomalies */ async getAnomalies(filters = {}) { const { status = 'open', severity, type, limit = 100, offset = 0 } = filters; let whereClause = ['status = ?']; let params = [status]; if (severity) { whereClause.push('severity = ?'); params.push(severity); } if (type) { whereClause.push('type = ?'); params.push(type); } params.push(limit, offset); return new Promise((resolve, reject) => { db.all( `SELECT * FROM security_anomalies WHERE ${whereClause.join(' AND ')} ORDER BY created_at DESC LIMIT ? OFFSET ?`, params, (err, rows) => { if (err) reject(err); else resolve(rows); } ); }); } /** * Resolve anomaly */ async resolveAnomaly(anomalyId, resolvedBy, notes) { return new Promise((resolve, reject) => { db.run( `UPDATE security_anomalies SET status = 'resolved', resolved_at = CURRENT_TIMESTAMP, resolved_by = ?, resolution_notes = ? WHERE anomaly_id = ?`, [resolvedBy, notes, anomalyId], (err) => { if (err) reject(err); else { logger.info(`[SecurityIntelligence] Anomaly resolved: ${anomalyId} by user ${resolvedBy}`); resolve(); } } ); }); } /** * Get threat intelligence */ async getThreatIntelligence(filters = {}) { const { level, type, limit = 100 } = filters; let whereClause = []; let params = []; if (level) { whereClause.push('threat_level = ?'); params.push(level); } if (type) { whereClause.push('indicator_type = ?'); params.push(type); } const where = whereClause.length > 0 ? `WHERE ${whereClause.join(' AND ')}` : ''; params.push(limit); return new Promise((resolve, reject) => { db.all( `SELECT * FROM threat_intelligence ${where} ORDER BY last_seen DESC, occurrence_count DESC LIMIT ?`, params, (err, rows) => { if (err) reject(err); else resolve(rows); } ); }); } /** * Get security intelligence dashboard data */ async getDashboardData() { const [anomalies, threats, score] = await Promise.all([ this.getAnomalies({ status: 'open', limit: 50 }), this.getThreatIntelligence({ limit: 20 }), this.calculateThreatScore() ]); const anomalyStats = { critical: anomalies.filter(a => a.severity === 'critical').length, high: anomalies.filter(a => a.severity === 'high').length, medium: anomalies.filter(a => a.severity === 'medium').length, low: anomalies.filter(a => a.severity === 'low').length }; return { threatScore: score, anomalies: anomalies.slice(0, 10), anomalyStats, threats: threats.slice(0, 10), timestamp: new Date().toISOString() }; } } // Create singleton instance const securityIntelligence = new SecurityIntelligence(); module.exports = securityIntelligence;