Modified Z = 0.6745 * (x − median) / MAD

ערכים מעל 3.5 נחשבים חריגים (Iglewicz & Hoaglin, 1993)

עמיאל פלד | 0549988754

import numpy as np
from scipy import stats
import pandas as pd

# Simulated network traffic: packet sizes (bytes)
np.random.seed(42)
normal_traffic = np.random.normal(loc=500, scale=50, size=1000)
# Inject anomalies (large packets, possible exfiltration)
anomalies = np.array([2000, 2500, 1800, 3000])
data = np.concatenate([normal_traffic, anomalies])

df = pd.DataFrame({'packet_size': data})

# Compute Z-Score
df['z_score'] = np.abs(stats.zscore(df['packet_size']))

# Flag anomalies with threshold = 3
threshold = 3
df['is_anomaly'] = df['z_score'] > threshold

print(f"Detected {df['is_anomaly'].sum()} anomalies")
print(df[df['is_anomaly']][['packet_size', 'z_score']])

עמיאל פלד | 0549988754

import numpy as np
import pandas as pd

# Simulated connection counts per second
np.random.seed(42)
normal_conn = np.random.poisson(lam=20, size=1000)
# Inject anomalies (DDoS-like spike)
anomalies = np.array([500, 750, 600, 480])
data = np.concatenate([normal_conn, anomalies])

df = pd.DataFrame({'connections': data})

# Compute IQR bounds
Q1 = df['connections'].quantile(0.25)
Q3 = df['connections'].quantile(0.75)
IQR = Q3 - Q1

lower = Q1 - 1.5 * IQR
upper = Q3 + 1.5 * IQR

df['is_anomaly'] = (
    (df['connections'] < lower) |
    (df['connections'] > upper)
)

print(f"Q1={Q1}, Q3={Q3}, IQR={IQR}")
print(f"Bounds: [{lower:.1f}, {upper:.1f}]")
print(f"Detected {df['is_anomaly'].sum()} anomalies")

עמיאל פלד | 0549988754

import numpy as np
import pandas as pd
from scipy import stats

# Generate dataset with mixed anomalies
np.random.seed(0)
normal = np.random.normal(loc=300, scale=40, size=500)
anomalies = np.array([1000, 950, 1100, -200, 1200])
data = pd.Series(np.concatenate([normal, anomalies]), name='metric')

results = pd.DataFrame({'value': data})

# --- Method 1: Z-Score ---
results['z_score'] = np.abs(stats.zscore(data))
results['zscore_flag'] = results['z_score'] > 3

# --- Method 2: IQR ---
Q1, Q3 = data.quantile(0.25), data.quantile(0.75)
IQR = Q3 - Q1
results['iqr_flag'] = (data < Q1 - 1.5*IQR) | (data > Q3 + 1.5*IQR)

# --- Method 3: Modified Z-Score (robust) ---
median = np.median(data)
mad = np.median(np.abs(data - median))
results['mod_z'] = 0.6745 * np.abs(data - median) / mad
results['mod_z_flag'] = results['mod_z'] > 3.5

# Summary
print("=== Anomaly Detection Comparison ===")
print(f"Z-Score    detected: {results['zscore_flag'].sum()} anomalies")
print(f"IQR        detected: {results['iqr_flag'].sum()} anomalies")
print(f"Modified Z detected: {results['mod_z_flag'].sum()} anomalies")

# Show rows flagged by at least one method
flagged = results[
    results['zscore_flag'] | results['iqr_flag'] | results['mod_z_flag']
]
print("\nFlagged records:")
print(flagged[['value','zscore_flag','iqr_flag','mod_z_flag']])

עמיאל פלד | 0549988754

import tensorflow as tf

encoder = tf.keras.Sequential([
    tf.keras.layers.Dense(64, activation='relu'),
    tf.keras.layers.Dense(32, activation='relu'),
    tf.keras.layers.Dense(16, activation='relu')
])
decoder = tf.keras.Sequential([
    tf.keras.layers.Dense(32, activation='relu'),
    tf.keras.layers.Dense(64, activation='relu'),
    tf.keras.layers.Dense(input_dim, activation='sigmoid')
])

עמיאל פלד | 0549988754

pip install tensorflow numpy

import numpy as np
import tensorflow as tf
from tensorflow.keras import layers, Model

# Build Autoencoder
input_layer = layers.Input(shape=(input_dim,))
encoded = layers.Dense(64, activation='relu')(input_layer)
encoded = layers.Dense(32, activation='relu')(encoded)
latent = layers.Dense(16, activation='relu')(encoded)
decoded = layers.Dense(32, activation='relu')(latent)
decoded = layers.Dense(64, activation='relu')(decoded)
output_layer = layers.Dense(input_dim,
                activation='sigmoid')(decoded)

autoencoder = Model(input_layer, output_layer)
autoencoder.compile(optimizer='adam', loss='mse')

עמיאל פלד | 0549988754

pip install tensorflow numpy

# Train ONLY on normal traffic
autoencoder.fit(X_normal, X_normal,
    epochs=50, batch_size=256,
    validation_split=0.1)

# Detect anomalies — compute reconstruction error
reconstructions = autoencoder.predict(X_test)
mse = np.mean(np.power(X_test - reconstructions, 2), axis=1)

# Set threshold at 95th percentile of training errors
threshold = np.percentile(mse, 95)
anomalies = mse > threshold

עמיאל פלד | 0549988754

pip install alibi-detect

from alibi_detect.od import IForest, OutlierAE
from alibi_detect.utils.saving import save_detector

# Autoencoder Outlier Detector
od = OutlierAE(threshold=0.012,
               encoder_net=encoder,
               decoder_net=decoder)
od.fit(X_train, epochs=30)

# Save for production
save_detector(od, './anomaly_detector')

# Predict with metadata
result = od.predict(X_test,
    outlier_type='instance',
    return_feature_score=True)

עמיאל פלד | 0549988754

pip install scikit-learn matplotlib

from sklearn.metrics import (
    classification_report,
    roc_auc_score,
    confusion_matrix,
    precision_recall_curve
)
import matplotlib.pyplot as plt

# Evaluate model
y_pred = model.predict(X_test)
y_scores = model.decision_function(X_test)

print(classification_report(y_test, y_pred,
    target_names=['Normal', 'Anomaly']))

# AUC-ROC
auc = roc_auc_score(y_test, y_scores)
print(f"AUC-ROC: {auc:.4f}")

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
print(f"False Positives: {cm[0][1]}")
print(f"False Negatives: {cm[1][0]}")
print(f"True Positives:  {cm[1][1]}")

עמיאל פלד | 0549988754

pip install pandas scikit-learn

import pandas as pd
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import LabelEncoder

# Load user activity logs
logs = pd.read_csv('user_activity.csv')

# Feature engineering per user session
features = logs.groupby(['user_id', 'session_id']).agg({
    'hour':        'first',
    'country':     'first',
    'data_bytes':  'sum',
    'resource':    'nunique',
    'failed_auth': 'sum'
}).reset_index()

# Encode categorical features
le = LabelEncoder()
features['country_enc'] = le.fit_transform(features['country'])

עמיאל פלד | 0549988754

pip install pandas scikit-learn

# Train per-user baseline (unsupervised)
model = IsolationForest(contamination=0.03)
model.fit(features[numeric_cols])

# Risk score (-1 to 0 range, lower = riskier)
risk_scores = model.decision_function(features[numeric_cols])
features['risk'] = risk_scores

# Flag high-risk sessions
high_risk = features[features['risk'] < -0.1]
print(f"High-risk sessions: {len(high_risk)}")
print(high_risk[['user_id', 'hour', 'data_bytes', 'risk']].sort_values('risk'))

עמיאל פלד | 0549988754

pip install torch-geometric
pip install torch numpy

import torch
from torch_geometric.nn import GCNConv
from torch_geometric.data import Data

# Build network graph from connections
edge_index = torch.tensor(
    [[src_ids], [dst_ids]], dtype=torch.long)
x = torch.tensor(node_features, dtype=torch.float)

data = Data(x=x, edge_index=edge_index)

# GCN for anomaly scoring
class GraphAnomalyDetector(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = GCNConv(in_channels, 64)
        self.conv2 = GCNConv(64, 32)
        self.score  = torch.nn.Linear(32, 1)

עמיאל פלד | 0549988754

pip install numpy scipy

import numpy as np
from scipy import stats

def detect_beaconing(connection_times, threshold=0.1):
    """Detect periodic communication patterns."""
    intervals = np.diff(sorted(connection_times))
    
    if len(intervals) < 5:
        return False, 0
    
    # Low coefficient of variation = very regular
    cv = intervals.std() / intervals.mean()
    
    # Check periodicity with FFT
    fft = np.fft.fft(intervals)
    dominant_freq = np.argmax(np.abs(fft[1:])) + 1
    
    is_beacon = cv < threshold
    return is_beacon, cv

# Alert if beaconing detected
for host, times in host_connections.items():
    is_beacon, score = detect_beaconing(times)
    if is_beacon:
        print(f"ALERT: {host} beaconing (CV={score:.3f})")

עמיאל פלד | 0549988754

pip install elasticsearch

from elasticsearch import Elasticsearch
from datetime import datetime
import json

es = Elasticsearch(['http://localhost:9200'])

def send_alert(anomaly_data, score, explanation):
    """Send anomaly alert to Elasticsearch/SIEM."""
    alert = {
        "@timestamp": datetime.utcnow().isoformat(),
        "event.category": "network",
        "event.type": "anomaly",
        "anomaly.score": float(score),
        "anomaly.explanation": explanation,
        "source.ip": anomaly_data['src_ip'],
        "destination.ip": anomaly_data['dst_ip'],
        "network.bytes": anomaly_data['bytes'],
        "severity": "high" if score > 0.8 else "medium"
    }
    
    es.index(
        index=f"anomalies-{datetime.now():%Y.%m.%d}",
        document=alert
    )
    print(f"Alert sent: {anomaly_data['src_ip']}")

עמיאל פלד | 0549988754

from modAL.models import ActiveLearner
from modAL.uncertainty import uncertainty_sampling

learner = ActiveLearner(
    estimator=IsolationForest(contamination=0.05),
    query_strategy=uncertainty_sampling,
    X_training=X_initial,
    y_training=y_initial
)

# Query the most informative samples
query_idx, query_sample = learner.query(X_pool, n_instances=10)
# Analyst labels these 10 samples
y_new = analyst_review(query_sample)
learner.teach(X_pool[query_idx], y_new)

עמיאל פלד | 0549988754

pip install pyod

import json
import subprocess
from pyod.models.iforest import IForest

def process_suricata_alert(alert_json):
    """Enrich Suricata alert with ML anomaly score."""
    alert = json.loads(alert_json)
    
    features = extract_flow_features(alert)
    score = ml_model.decision_function([features])[0]
    
    # Enrich alert with ML score
    alert['ml_anomaly_score'] = float(score)
    alert['ml_is_anomaly'] = score < threshold
    alert['ml_confidence'] = abs(score)
    
    return alert

# Tail Suricata EVE JSON log
with open('/var/log/suricata/eve.json', 'r') as f:
    for line in follow(f):
        enriched = process_suricata_alert(line)
        if enriched['ml_is_anomaly']:
            send_high_priority_alert(enriched)

עמיאל פלד | 0549988754

# Dockerfile for anomaly detection service
FROM python:3.11-slim

WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

COPY . .

EXPOSE 8080

CMD ["uvicorn", "main:app", 
     "--host", "0.0.0.0", 
     "--port", "8080"]

# docker-compose.yml
version: '3.8'
services:
  anomaly-detector:
    build: .
    ports: ["8080:8080"]
    volumes:
      - ./models:/app/models
    environment:
      - MODEL_PATH=/app/models/isolation_forest.pkl
      - THRESHOLD=0.05
      - KAFKA_BROKER=kafka:9092

  kafka:
    image: confluentinc/cp-kafka:latest
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

  elasticsearch:
    image: elasticsearch:8.8.0

עמיאל פלד | 0549988754

pip install fastapi uvicorn pydantic scikit-learn numpy

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import pickle
import numpy as np

app = FastAPI(title="Network Anomaly Detection API")

# Load pre-trained model
with open('model.pkl', 'rb') as f:
    model = pickle.load(f)

class NetworkFlow(BaseModel):
    src_bytes: float
    dst_bytes: float
    duration: float
    protocol: int
    src_port: int
    dst_port: int

class AnomalyResult(BaseModel):
    is_anomaly: bool
    score: float
    severity: str

@app.post("/predict", response_model=AnomalyResult)
async def predict_anomaly(flow: NetworkFlow):
    features = np.array([[
        flow.src_bytes, flow.dst_bytes,
        flow.duration, flow.protocol,
        flow.src_port, flow.dst_port
    ]])
    score = model.decision_function(features)[0]
    is_anomaly = score < -0.1
    severity = "high" if score < -0.5 else "medium"
    return AnomalyResult(is_anomaly=is_anomaly,
        score=float(score), severity=severity)

עמיאל פלד | 0549988754

pip install flwr  # Flower - Federated Learning

import flwr as fl
import numpy as np

class AnomalyClient(fl.client.NumPyClient):
    def get_parameters(self, config):
        return model.get_weights()
    
    def fit(self, parameters, config):
        model.set_weights(parameters)
        model.fit(local_X_train, local_X_train,
                  epochs=1, batch_size=256)
        return model.get_weights(), len(local_X_train), {}
    
    def evaluate(self, parameters, config):
        model.set_weights(parameters)
        loss = model.evaluate(local_X_test, local_X_test)
        return loss, len(local_X_test), {}

fl.client.start_numpy_client(
    server_address="federated-server:8080",
    client=AnomalyClient()
)