anti_parasite.py

#!/usr/bin/env python3
"""
SentinelAI · Anti-Parasite Engine
=================================
Détecte et repousse les contenus indésirables / non qualitatifs dans Google
via un modèle BERT/RoBERTa multilingue fine-tuné.

Ports :
    - API Flask : 5001 (conflit évité avec MCP 5000)
    - Modèle de base : distilbert-base-multilingual-cased

Routes :
    /health          -> santé du service
    /train           -> fine-tuning sur données d'exemple
    /classify        -> classification d'un texte/URL
    /batch           -> classification batch
    /dilute          -> génération de contre-contenu positif
"""

import os
import json
import time
import hashlib
import signal
import subprocess
from datetime import datetime
from typing import List, Dict

from flask import Flask, request, jsonify
from flask_cors import CORS
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification,
    Trainer,
    TrainingArguments,
    pipeline,
)
import torch
from torch.utils.data import Dataset

# ── Configuration ──────────────────────────────────────────────────────────
MODEL_NAME = "distilbert-base-multilingual-cased"
MODEL_DIR = "./models/anti_parasite"
LABELS = {0: "parasite", 1: "high_quality"}
ID2LABEL = {0: "parasite", 1: "high_quality"}
LABEL2ID = {"parasite": 0, "high_quality": 1}

# ── App ──────────────────────────────────────────────────────────────────────
app = Flask(__name__)
CORS(app)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tokenizer = None
model = None


# ── Dataset minimal d'entraînement ───────────────────────────────────────────
class ParasiteDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_length=128):
        self.encodings = tokenizer(texts, truncation=True, padding=True, max_length=max_length)
        self.labels = labels

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item["labels"] = torch.tensor(self.labels[idx])
        return item


# ── Données d'exemple pour démonstration ─────────────────────────────────────
DEFAULT_TRAIN_DATA = {
    "texts": [
        # Parasite / indésirable
        "Buy cheap viagra now!!! Click here!!! Free money 100% guaranteed",
        "URGENT: You won a lottery. Send your bank details immediately",
        "Keyword stuffing SEO best SEO cheap SEO top SEO ranking fast",
        "Download free movie pirated HD no virus click here now",
        "Cette arnaque vous fait gagner 1000€ par jour sans effort",
        "🔥🔥🔥 CLIQUEZ ICI !!! OFFRE EXCLUSIVE LIMITÉE !!! 🔥🔥🔥",
        "Duplicate content copy paste duplicate content same article",
        "Hidden links porn casino viagra hidden text white on white",
        "Make money online fast working from home guaranteed $5000",
        "Recette de gâteau recette de gâteau recette de gâteau",
        # Haute qualité
        "The treaty of Westphalia established the modern international system of sovereign states in 1648",
        "A comprehensive guide to renewable energy sources and their impact on climate change",
        "L'article 9 du Code civil protège le droit à la vie privée et le droit à l'image",
        "Research published in Nature demonstrates that mRNA vaccines are safe and effective",
        "Le guide pratique pour comprendre le RGPD et ses obligations pour les entreprises",
        "Tutorial: how to implement secure authentication with OAuth 2.0 and OpenID Connect",
        "Analyse comparative des approches pédagogiques en éducation inclusive",
        "The European Court of Justice ruling on the right to be forgotten in Google Spain v AEPD",
        "Mode d'emploi complet pour configurer un serveur web sécurisé sous Linux",
        "Best practices for content moderation and trust & safety operations",
    ],
    "labels": [
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    ]
}


_MODEL_LOADING = False
_MODEL_ERROR = None

# ── Chargement du modèle ─────────────────────────────────────────────────────
def load_model():
    global tokenizer, model, _MODEL_LOADING, _MODEL_ERROR
    if tokenizer is not None and model is not None:
        return
    if _MODEL_LOADING:
        raise RuntimeError("Modèle en cours de chargement, réessayez dans quelques secondes")
    if _MODEL_ERROR:
        raise RuntimeError(f"Modèle indisponible: {_MODEL_ERROR}")

    _MODEL_LOADING = True
    print(f"[Anti-Parasite] Chargement du modèle ({MODEL_NAME}) sur {device} ...")

    try:
        if os.path.exists(MODEL_DIR) and os.path.isdir(MODEL_DIR):
            print(f"[Anti-Parasite] Chargement du modèle fine-tuné depuis {MODEL_DIR}")
            tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
            model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR)
        else:
            print(f"[Anti-Parasite] Téléchargement du modèle de base {MODEL_NAME}")
            tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
            model = AutoModelForSequenceClassification.from_pretrained(
                MODEL_NAME,
                num_labels=2,
                id2label=ID2LABEL,
                label2id=LABEL2ID,
            )
        model.to(device)
        model.eval()
    except Exception as e:
        _MODEL_ERROR = str(e)
        print(f"[Anti-Parasite] ERREUR chargement modèle: {e}")
        raise
    finally:
        _MODEL_LOADING = False


def _fallback_predict(texts: List[str]) -> List[Dict]:
    """
    Mode de secours si torch/transformers n'est pas installé ou si le
    téléchargement a échoué. Renvoie un score heuristique basé sur des mots-clés
    connus de contenu parasite.
    """
    parasite_signals = [
        "viagra", "cialis", "casino", "lottery", "loterie", "click here",
        "cliquez ici", "urgent", "gagner", "free money", "argent gratuit",
        "crypto", "bitcoin", "mlm", "arnaqu", "escro", "arnaqué", "fraude",
        "duplicate content", "keyword stuffing", "free download", "porn",
        "xxx", "hidden links", "buy now", "limited offer", "offre limitée",
        "100% guaranteed", "guaranteed", "make money online", "no virus",
        "pirated", "𝙩𝙚𝙧𝙢𝙞𝙣𝙚", "????", "!!!", "🔥", "💰", "💸",
    ]
    results = []
    for text in texts:
        lowered = text.lower()
        score = 0
        for sig in parasite_signals:
            if sig in lowered:
                score += 1
                if sig in ["arnaqu", "escro", "fraude", "pirated", "viagra", "casino", "crypto", "porn", "xxx"]:
                    score += 1
        max_score = max(len(parasite_signals) * 0.15, 5)
        parasite_score = min(score / max_score, 1.0)
        label_id = 1 if parasite_score < 0.5 else 0

        results.append({
            "text": text,
            "label": ID2LABEL[label_id],
            "label_id": label_id,
            "confidence": round(0.6 + abs(0.5 - parasite_score) * 0.7, 4),
            "parasite_score": round(parasite_score, 4),
            "quality_score": round(1 - parasite_score, 4),
            "fallback": True,
        })
    return results


# ── Entraînement ─────────────────────────────────────────────────────────────
@app.route("/train", methods=["POST"])
def train():
    data = request.get_json(force=True) or {}
    texts = data.get("texts", DEFAULT_TRAIN_DATA["texts"])
    labels = data.get("labels", DEFAULT_TRAIN_DATA["labels"])

    if len(texts) != len(labels):
        return jsonify({"error": "texts and labels must have same length"}), 400

    os.makedirs(MODEL_DIR, exist_ok=True)

    tok = AutoTokenizer.from_pretrained(MODEL_NAME)
    mdl = AutoModelForSequenceClassification.from_pretrained(
        MODEL_NAME,
        num_labels=2,
        id2label=ID2LABEL,
        label2id=LABEL2ID,
    )

    train_dataset = ParasiteDataset(texts, labels, tok)

    args = TrainingArguments(
        output_dir="./results/anti_parasite",
        num_train_epochs=3,
        per_device_train_batch_size=8,
        per_device_eval_batch_size=8,
        warmup_steps=50,
        weight_decay=0.01,
        logging_dir="./logs/anti_parasite",
        logging_steps=5,
        save_strategy="no",
        report_to="none",
    )

    trainer = Trainer(
        model=mdl,
        args=args,
        train_dataset=train_dataset,
    )

    trainer.train()

    # Sauvegarde
    mdl.save_pretrained(MODEL_DIR)
    tok.save_pretrained(MODEL_DIR)

    global tokenizer, model
    tokenizer = tok
    model = mdl
    model.to(device)
    model.eval()

    return jsonify({
        "status": "trained",
        "samples": len(texts),
        "model_dir": MODEL_DIR,
        "timestamp": datetime.utcnow().isoformat(),
    })


# ── Classification ────────────────────────────────────────────────────────────
def predict(texts: List[str]) -> List[Dict]:
    try:
        load_model()
    except Exception:
        # Si torch/transformers n'est pas dispo, on utilise l'heuristique
        if tokenizer is None or model is None:
            return _fallback_predict(texts)
        raise

    try:
        inputs = tokenizer(texts, return_tensors="pt", truncation=True, padding=True, max_length=256)
        inputs = {k: v.to(device) for k, v in inputs.items()}

        with torch.no_grad():
            outputs = model(**inputs)
            logits = outputs.logits
            probs = torch.softmax(logits, dim=1)
            predicted_ids = torch.argmax(logits, dim=1)

        results = []
        for i, text in enumerate(texts):
            label_id = predicted_ids[i].item()
            prob = probs[i][label_id].item()
            results.append({
                "text": text,
                "label": ID2LABEL[label_id],
                "label_id": label_id,
                "confidence": round(prob, 4),
                "parasite_score": round(probs[i][0].item(), 4),
                "quality_score": round(probs[i][1].item(), 4),
                "fallback": False,
            })
        return results
    except Exception as e:
        # Dernier recours: heuristique pour ne jamais crasher le front
        return _fallback_predict(texts)


@app.route("/classify", methods=["POST"])
def classify():
    data = request.get_json(force=True)
    text = data.get("text", "").strip()
    if not text:
        return jsonify({"error": "No text provided"}), 400

    try:
        results = predict([text])
        if not results:
            return jsonify({"error": "Prediction returned empty"}), 500
        result = results[0]
    except Exception as e:
        return jsonify({"error": f"Prediction failed: {str(e)}"}), 500

    result["timestamp"] = datetime.utcnow().isoformat()
    result["hash"] = hashlib.sha256(text.encode()).hexdigest()[:16]
    return jsonify(result)


@app.route("/batch", methods=["POST"])
def batch():
    data = request.get_json(force=True)
    texts = data.get("texts", [])
    if not texts or not isinstance(texts, list):
        return jsonify({"error": "Provide a list of texts"}), 400

    results = predict(texts)
    return jsonify({
        "results": results,
        "count": len(results),
        "parasite_count": sum(1 for r in results if r["label"] == "parasite"),
        "high_quality_count": sum(1 for r in results if r["label"] == "high_quality"),
        "timestamp": datetime.utcnow().isoformat(),
    })


# ── Dilution sémantique ──────────────────────────────────────────────────────
@app.route("/dilute", methods=["POST"])
def dilute():
    """
    Génère des contenus positifs/neutres pour diluer un cluster vectoriel négatif.
    Utilise le modèle comme assistant ; en production, remplacer par un générateur.
    """
    data = request.get_json(force=True)
    keyword = data.get("keyword", "").strip()
    context = data.get("context", "").strip()
    if not keyword:
        return jsonify({"error": "No keyword provided"}), 400

    # Contenu positif pré-généré (template SEO)
    positive_snippets = [
        f"Guide complet et sourcé sur {keyword} : définitions, cadre légal et bonnes pratiques.",
        f"{keyword} : analyse factuelle, ressources officielles et contacts utiles.",
        f"Comprendre {keyword} en 5 minutes — version vérifiée par des experts.",
        f"FAQ {keyword} : réponses aux questions les plus fréquentes.",
        f"Dossier {keyword} : chiffres clés, évolution réglementaire et impacts.",
    ]

    if context:
        positive_snippets.append(
            f"Retour d'expérience et mise au point concernant {keyword} : {context[:200]}"
        )

    return jsonify({
        "keyword": keyword,
        "context": context,
        "positive_contents": positive_snippets,
        "strategy": "Semantic dilution — flood positive/neutral content to push parasite results down",
        "timestamp": datetime.utcnow().isoformat(),
    })


# ── Anti-subprocess / sécurité basique ───────────────────────────────────────
@app.route("/processes", methods=["GET"])
def suspicious_processes():
    """Liste les processus suspects (démo Linux/macOS)."""
    suspicious = []
    try:
        result = subprocess.run(
            ["ps", "-eo", "pid,comm,args"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        for line in result.stdout.splitlines()[1:]:
            lowered = line.lower()
            if any(k in lowered for k in ["miner", "keylogger", "trojan", "backdoor", "exploit", "xmrig"]):
                suspicious.append(line.strip())
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)}), 500

    return jsonify({
        "suspicious_processes": suspicious,
        "count": len(suspicious),
        "timestamp": datetime.utcnow().isoformat(),
    })


@app.route("/health", methods=["GET"])
def health():
    return jsonify({
        "status": "ok",
        "model": MODEL_NAME,
        "model_dir_exists": os.path.exists(MODEL_DIR),
        "device": str(device),
        "timestamp": datetime.utcnow().isoformat(),
    })


# ── Main ─────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
    print("🚀 SentinelAI Anti-Parasite Engine running on http://0.0.0.0:5001")
    print("   Chargement du modèle DistilBERT se fera au premier appel /classify")
    app.run(host="0.0.0.0", port=5001, debug=True, use_reloader=False)