Add application file

README.md

@@ -1,14 +1,108 @@
+# 🧠 Community Rating System for Social Posts
+
+A machine learning-powered system that evaluates **community comments or notes** on public posts for **credibility, helpfulness, and factual consistency**, while actively resisting manipulation like spam voting or coordinated misinformation.
+
 ---
-title: Community Rating System
-emoji: 👀
-colorFrom: green
-colorTo: red
-sdk: gradio
-sdk_version: 5.32.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: AI rates comments for helpfulness and trustworthiness.
+
+## 🔍 Project Overview
+
+This project builds a full-stack pipeline that:
+
+- Processes public comments (e.g., Reddit, StackOverflow)
+- Extracts features like sentiment, toxicity, readability, and named entities
+- Scores notes/comments for **credibility** and **helpfulness**
+- Detects manipulation attempts (e.g., bot voting, duplicate spam)
+- Provides a web interface to interactively test the system
+
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## 🚀 Features
+
+- ✅ Transformer-based helpfulness scoring (e.g., RoBERTa)
+- ✅ Anomaly detection using Isolation Forest
+- ✅ Sentiment and toxicity classification
+- ✅ Interactive UI with Gradio or Streamlit
+- ✅ REST API using FastAPI
+- ✅ Dockerized and deployable on Hugging Face Spaces or Render
+
+---
+
+## 🗂️ Project Structure
+
+```plaintext
+.
+├── data/
+│   └── sample_comments.csv          # Input dataset
+├── models/                          # Saved model checkpoints
+├── src/
+│   ├── app.py                       # Web interface
+│   ├── anomaly.py                   # Anomaly detection logic
+│   ├── data_preprocessing.py        # Cleaning, tokenization, features
+│   ├── dataset.py                   # PyTorch dataset logic
+│   ├── evaluate.py                  # Evaluation metrics
+│   ├── main.py                      # Entry script
+│   ├── manipulation_detect.py       # Bot/spam detection
+│   ├── model.py                     # Helpfulness scoring model
+│   ├── train_model.py               # Training loop
+│   └── utils.py                     # Helper functions
+├── requirements.txt                 # Dependencies
+├── dockerfile                       # Container setup
+└── README.md                        
+
+## 📦 Setup Instructions
+🔧 1. Install Dependencies
+- git clone https://github.com/yourusername/community-rating-system.git
+- cd community-rating-system
+- python -m venv env
+- source env/bin/activate  # or .\env\Scripts\activate on Windows
+- pip install -r requirements.txt
+
+## 🧪 2. Train the Helpfulness Model
+- python src/train_model.py
+
+## 🧬 3. Run Anomaly Detection
+- python src/manipulation_detect.py
+
+## 🌐 4. Launch Web Interface
+- python src/app.py
+or
+- uvicorn src.main:app --reload
+
+## 🔐 Manipulation Resistance
+- This project uses:
+- Isolation Forest to detect abnormal user behavior (e.g., sudden karma spikes)
+- Text similarity analysis to flag near-duplicate spam
+- (Optional) Adversarial training to improve robustness against vote brigading
+
+## 🧠 ML Model Architecture
+- Backbone: RoBERTa-base or BERT-base
+- Input: comment_text, along with metadata like upvotes, length, time, etc.
+- Output: A scalar score (0.0–1.0) representing helpfulness or credibility
+
+Additional Features:
+✅ Sentiment polarity
+
+✅ Toxicity score (via Detoxify)
+
+✅ Readability score (Flesch–Kincaid, Gunning Fog Index)
+
+✅ Named Entity Types (political, scientific, health-related)
+
+
+## 🧪 Evaluation
+Key metrics used for evaluating the scoring model and resistance components:
+
+🏆 ROC-AUC
+
+🎯 Precision@K
+
+🧩 Agreement with majority vote labels
+
+📊 Confusion Matrix on manipulated vs. organic comment behavior
+
+## 📦 Docker Deployment
+- To build and run the container locally:
+- docker build -t community-rating-system .
+- docker run -p 7860:7860 community-rating-system
+
+

dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY ./src ./src
+COPY ./data ./data
+COPY ./models ./models
+
+EXPOSE 7860
+
+CMD ["python", "src/app.py"]

models/roberta_metadata_classifier.py

@@ -0,0 +1,28 @@
+import torch
+import torch.nn as nn
+from transformers import RobertaModel
+
+class RobertaWithMetadata(nn.Module):
+    def __init__(self, model_name='roberta-base', metadata_dim=3, dropout=0.3):
+        super(RobertaWithMetadata, self).__init__()
+        self.roberta = RobertaModel.from_pretrained(model_name)
+        self.dropout = nn.Dropout(dropout)
+        
+        hidden_size = self.roberta.config.hidden_size  # usually 768
+        self.metadata_dim = metadata_dim
+
+        # Final classifier takes RoBERTa CLS output + metadata
+        self.classifier = nn.Sequential(
+            nn.Linear(hidden_size + metadata_dim, 256),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(256, 1)  # single score output (e.g., helpfulness)
+        )
+
+    def forward(self, input_ids, attention_mask, metadata):
+        roberta_out = self.roberta(input_ids=input_ids, attention_mask=attention_mask)
+        cls_output = roberta_out.pooler_output  # [CLS] token representation
+
+        combined = torch.cat((cls_output, metadata), dim=1)
+        out = self.classifier(self.dropout(combined))
+        return out.squeeze(1)  # final regression output

requirements.txt

@@ -0,0 +1,13 @@
+torch
+transformers
+datasets
+scikit-learn
+pyod
+gradio
+pandas
+numpy
+detoxify
+textstat
+fastapi
+uvicorn
+joblib

src/anomaly.py

@@ -0,0 +1,9 @@
+import joblib
+import numpy as np
+
+def load_anomaly_model(path='models/anomaly_detector.pkl'):
+    return joblib.load(path)
+
+def is_anomalous(features, model):
+    score = model.decision_function([features])[0]
+    return model.predict([features])[0] == -1, score

src/app.py

@@ -0,0 +1,68 @@
+import gradio as gr
+import torch
+from transformers import RobertaTokenizer
+from src.model import CommentClassifier
+from src.utils import preprocess_comment, extract_metadata_features
+from src.anomaly import load_anomaly_model
+
+
+# Load tokenizer and model
+tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
+model = CommentClassifier()
+model.load_state_dict(torch.load("models/best_model.pt", map_location=torch.device("cpu")))
+model.eval()
+
+# Load Isolation Forest model
+anomaly_model = load_anomaly_model()
+
+def predict_comment(text):
+    if not text.strip():
+        return "Enter a valid comment!", 0, 0, 0, 0
+
+    # Preprocess
+    input_text = preprocess_comment(text)
+    meta_features = extract_metadata_features(text)
+    meta_tensor = torch.tensor(meta_features, dtype=torch.float).unsqueeze(0)
+
+    # Tokenize
+    inputs = tokenizer(
+        input_text, padding="max_length", truncation=True, max_length=128, return_tensors="pt"
+    )
+
+    with torch.no_grad():
+        outputs = model(
+            input_ids=inputs["input_ids"],
+            attention_mask=inputs["attention_mask"],
+            meta_features=meta_tensor
+        )
+        probs = torch.softmax(outputs, dim=1).squeeze().numpy()
+
+    labels = ["Negative", "Neutral", "Positive"]
+    prediction = labels[probs.argmax()]
+    sentiment_score = round(probs[2], 2)
+    
+    # Fake/toxic/helpfulness scores (simple heuristics)
+    toxicity = float(meta_features[0])  # e.g., from Detoxify or rule-based
+    helpfulness = float(meta_features[2])
+    anomaly_score = anomaly_model.decision_function([meta_features])[0]
+
+    return prediction, sentiment_score, round(toxicity, 2), round(helpfulness, 2), round(anomaly_score, 2)
+
+# Gradio UI
+iface = gr.Interface(
+    fn=predict_comment,
+    inputs=gr.Textbox(lines=4, label="Enter a Comment"),
+    outputs=[
+        gr.Textbox(label="Predicted Sentiment"),
+        gr.Number(label="Sentiment Score (0-1)"),
+        gr.Number(label="Toxicity Score"),
+        gr.Number(label="Helpfulness Score"),
+        gr.Number(label="Anomaly Score")
+    ],
+    title="🧠 Comment Quality Analyzer",
+    description="Paste any social media comment and get its sentiment, toxicity, helpfulness, and anomaly scores using a RoBERTa-based ML model.",
+    allow_flagging="never",
+)
+
+if __name__ == "__main__":
+    iface.launch()

src/data_preprocessing.py

@@ -0,0 +1,50 @@
+import pandas as pd
+import numpy as np
+import re
+from sklearn.ensemble import IsolationForest
+from detoxify import Detoxify
+import textstat
+
+# Clean text
+def clean_text(text):
+    text = str(text).lower()
+    text = re.sub(r"http\S+|www\S+|https\S+", '', text)
+    text = re.sub(r'\@\w+|\#','', text)
+    text = re.sub(r'[^\w\s]', '', text)
+    return text
+
+# Load and preprocess
+def load_data(filepath):
+    df = pd.read_csv(filepath)
+    df.dropna(subset=["text", "helpfulness_score"], inplace=True)
+    return df
+
+def preprocess_text(df):
+    df['text'] = df['text'].apply(clean_text)
+
+    # Toxicity score
+    print("Computing toxicity scores...")
+    toxicity_results = Detoxify('original').predict(df['text'].tolist())
+    df['toxicity_score'] = toxicity_results['toxicity']
+
+    # Readability score (lower is harder to read)
+    print("Computing readability scores...")
+    df['readability_score'] = df['text'].apply(lambda x: textstat.flesch_reading_ease(x))
+
+    # Anomaly Detection
+    print("Running anomaly detection...")
+    meta_features = df[["toxicity_score", "readability_score"]].fillna(0)
+    clf = IsolationForest(contamination=0.05, random_state=42)
+    df['is_anomalous'] = clf.fit_predict(meta_features)
+    df['is_anomalous'] = df['is_anomalous'].apply(lambda x: 1 if x == -1 else 0)
+
+    return df
+
+# For custom evaluation later
+def compute_custom_metrics(y_true, y_pred):
+    from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+    return {
+        "MAE": mean_absolute_error(y_true, y_pred),
+        "MSE": mean_squared_error(y_true, y_pred),
+        "R2": r2_score(y_true, y_pred)
+    }

src/dataset.py

@@ -0,0 +1,37 @@
+# dataset.py
+
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+from transformers import RobertaTokenizer
+
+class CommentDataset(Dataset):
+    def __init__(self, data, tokenizer, max_len=128):
+        self.data = data
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        comment = self.data.iloc[idx]['comment']
+        label = self.data.iloc[idx]['label']
+        encoding = self.tokenizer(
+            comment,
+            padding='max_length',
+            truncation=True,
+            max_length=self.max_len,
+            return_tensors='pt'
+        )
+        return {
+            'input_ids': encoding['input_ids'].squeeze(),
+            'attention_mask': encoding['attention_mask'].squeeze(),
+            'label': torch.tensor(label, dtype=torch.long)
+        }
+
+def load_data(path):
+    df = pd.read_csv(path)
+    tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+    dataset = CommentDataset(df, tokenizer)
+    return dataset

src/evaluate.py

@@ -0,0 +1,54 @@
+import pandas as pd
+from sklearn.metrics import roc_auc_score, precision_score
+import torch
+from train_model import HelpfulnessModel, tokenizer
+import numpy as np
+from torch.utils.data import DataLoader
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+MODEL_PATH = "../models/helpfulness_model.pt"
+
+def evaluate():
+    df = pd.read_csv("../data/sample_comments.csv")
+
+    df['helpfulness'] = (df['score'] - df['score'].min()) / (df['score'].max() - df['score'].min() + 1e-8)
+
+    meta_cols = ['token_len', 'toxicity', 'readability', 'political_mentions',
+                 'health_mentions', 'science_mentions', 'engagement', 'time_since_posted']
+    X_meta = df[meta_cols].fillna(0).values
+
+    texts = df['clean_text'].tolist()
+    targets = df['helpfulness'].values
+
+    dataset = CommentDataset(texts, X_meta, targets)
+    dataloader = DataLoader(dataset, batch_size=16)
+
+    model = HelpfulnessModel("roberta-base", X_meta.shape[1])
+    model.load_state_dict(torch.load(MODEL_PATH, map_location=DEVICE))
+    model.to(DEVICE)
+    model.eval()
+
+    preds = []
+    actuals = []
+
+    with torch.no_grad():
+        for input_ids, attention_mask, meta_features, targets in dataloader:
+            input_ids = input_ids.to(DEVICE)
+            attention_mask = attention_mask.to(DEVICE)
+            meta_features = meta_features.to(DEVICE)
+
+            outputs = model(input_ids, attention_mask, meta_features)
+            preds.extend(outputs.cpu().numpy())
+            actuals.extend(targets.numpy())
+
+    auc = roc_auc_score(actuals, preds)
+    k = 100
+    df_eval = pd.DataFrame({'pred': preds, 'actual': actuals})
+    df_eval_sorted = df_eval.sort_values('pred', ascending=False).head(k)
+    precision_at_k = (df_eval_sorted['actual'] > 0.5).mean()
+
+    print(f"AUC: {auc:.4f}")
+    print(f"Precision@{k}: {precision_at_k:.4f}")
+
+if __name__ == "__main__":
+    evaluate()

src/main.py

@@ -0,0 +1,30 @@
+import argparse
+import os
+
+from train_model import train
+from evaluate import evaluate
+import app  
+
+def main():
+    parser = argparse.ArgumentParser(description="Community Comment Rating System")
+
+    parser.add_argument("--mode", type=str, required=True,
+                        choices=["train", "evaluate", "app"],
+                        help="Mode to run: train / evaluate / app")
+
+    args = parser.parse_args()
+
+    if args.mode == "train":
+        print("🚀 Starting Training...")
+        train()
+
+    elif args.mode == "evaluate":
+        print("📊 Starting Evaluation...")
+        evaluate()
+
+    elif args.mode == "app":
+        print("🌐 Launching Gradio App...")
+        app.iface.launch()
+
+if __name__ == "__main__":
+    main()

src/manipulation_detect.py

@@ -0,0 +1,40 @@
+import torch
+from torch.utils.data import Dataset
+from transformers import RobertaTokenizer
+
+class CommentDataset(Dataset):
+    def __init__(self, dataframe, max_len=128):
+        self.texts = dataframe["text"].tolist()
+        self.labels = dataframe["helpfulness_score"].tolist()
+        self.toxicity = dataframe["toxicity_score"].tolist()
+        self.readability = dataframe["readability_score"].tolist()
+        self.anomaly = dataframe["is_anomalous"].tolist()
+
+        self.tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
+        self.max_len = max_len
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        encoded = self.tokenizer(
+            self.texts[idx],
+            padding='max_length',
+            truncation=True,
+            max_length=self.max_len,
+            return_tensors="pt"
+        )
+
+        # Metadata features: concatenate scalar values
+        metadata = torch.tensor([
+            self.toxicity[idx],
+            self.readability[idx],
+            self.anomaly[idx]
+        ], dtype=torch.float)
+
+        return {
+            "input_ids": encoded["input_ids"].squeeze(0),
+            "attention_mask": encoded["attention_mask"].squeeze(0),
+            "metadata": metadata,
+            "label": torch.tensor(self.labels[idx], dtype=torch.float)
+        }

src/model.py

@@ -0,0 +1,16 @@
+import torch.nn as nn
+from transformers import RobertaModel
+
+class CommentClassifier(nn.Module):
+    def __init__(self, dropout=0.3):
+        super(CommentClassifier, self).__init__()
+        self.roberta = RobertaModel.from_pretrained("roberta-base")
+        self.dropout = nn.Dropout(dropout)
+        self.classifier = nn.Linear(self.roberta.config.hidden_size + 3, 1)  # +3 for metadata features
+
+    def forward(self, input_ids, attention_mask, metadata_features):
+        outputs = self.roberta(input_ids=input_ids, attention_mask=attention_mask)
+        pooled_output = outputs.pooler_output
+        combined = torch.cat((pooled_output, metadata_features), dim=1)
+        x = self.dropout(combined)
+        return self.classifier(x)

src/train_model.py

@@ -0,0 +1,96 @@
+import torch
+from torch import nn
+from torch.utils.data import DataLoader
+from transformers import RobertaModel, get_scheduler
+from torch.optim import AdamW
+from tqdm import tqdm
+import os
+
+from dataset import CommentDataset, load_data
+from model import CommentRatingModel
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+class RoBERTaWithMetadata(nn.Module):
+    def __init__(self, dropout=0.3):
+        super().__init__()
+        self.roberta = RobertaModel.from_pretrained("roberta-base")
+        self.dropout = nn.Dropout(dropout)
+        self.metadata_fc = nn.Linear(3, 64)
+        self.classifier = nn.Sequential(
+            nn.Linear(self.roberta.config.hidden_size + 64, 128),
+            nn.ReLU(),
+            nn.Dropout(0.2),
+            nn.Linear(128, 1)  # Regression output
+        )
+
+    def forward(self, input_ids, attention_mask, metadata):
+        roberta_out = self.roberta(input_ids=input_ids, attention_mask=attention_mask).pooler_output
+        meta_out = torch.relu(self.metadata_fc(metadata))
+        combined = torch.cat([roberta_out, meta_out], dim=1)
+        return self.classifier(combined)
+
+def train(model, train_loader, val_loader, epochs=5, lr=2e-5, checkpoint_path="models/best_model.pt"):
+    criterion = nn.MSELoss()
+    optimizer = AdamW(model.parameters(), lr=lr)
+    num_training_steps = epochs * len(train_loader)
+    scheduler = get_scheduler("linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps)
+
+    best_loss = float("inf")
+
+    for epoch in range(epochs):
+        model.train()
+        total_loss = 0
+        for batch in train_loader:
+            optimizer.zero_grad()
+            input_ids = batch["input_ids"].to(device)
+            attention_mask = batch["attention_mask"].to(device)
+            metadata = batch["metadata"].to(device)
+            labels = batch["label"].to(device).unsqueeze(1)
+
+            outputs = model(input_ids, attention_mask, metadata)
+            loss = criterion(outputs, labels)
+            loss.backward()
+            optimizer.step()
+            scheduler.step()
+
+            total_loss += loss.item()
+
+        avg_train_loss = total_loss / len(train_loader)
+
+        # Validation
+        model.eval()
+        val_loss = 0
+        with torch.no_grad():
+            for batch in val_loader:
+                input_ids = batch["input_ids"].to(device)
+                attention_mask = batch["attention_mask"].to(device)
+                metadata = batch["metadata"].to(device)
+                labels = batch["label"].to(device).unsqueeze(1)
+
+                outputs = model(input_ids, attention_mask, metadata)
+                loss = criterion(outputs, labels)
+                val_loss += loss.item()
+        avg_val_loss = val_loss / len(val_loader)
+
+        print(f"Epoch {epoch+1}: Train Loss = {avg_train_loss:.4f} | Val Loss = {avg_val_loss:.4f}")
+
+        if avg_val_loss < best_loss:
+            best_loss = avg_val_loss
+            torch.save(model.state_dict(), checkpoint_path)
+            print("✅ Saved best model")
+
+if __name__ == "__main__":
+    df = pd.read_csv("data/final_dataset.csv")
+    train_df = df.sample(frac=0.8, random_state=42)
+    val_df = df.drop(train_df.index)
+
+    train_dataset = CommentDataset(train_df)
+    val_dataset = CommentDataset(val_df)
+
+    train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=16)
+
+    model = RoBERTaWithMetadata().to(device)
+    os.makedirs("models", exist_ok=True)
+    train(model, train_loader, val_loader)

src/utils.py

@@ -0,0 +1,19 @@
+import numpy as np
+import re
+import string
+from detoxify import Detoxify
+
+detox_model = Detoxify('original')
+
+def preprocess_comment(text):
+    text = re.sub(r"http\S+", "", text)
+    text = text.lower()
+    text = text.translate(str.maketrans("", "", string.punctuation))
+    return text.strip()
+
+def extract_metadata_features(text):
+    toxicity = detox_model.predict(text)["toxicity"]
+    word_count = len(text.split())
+    readability = min(1.0, word_count / 50)  # Normalize to [0,1]
+    engagement = min(1.0, sum(1 for w in text.split() if len(w) > 6) / word_count) if word_count else 0
+    return np.array([toxicity, readability, engagement], dtype=np.float32)