src/processor.py

import os
import pandas as pd
import numpy as np
import torch
import joblib
import time
from datetime import datetime
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification

class Processor:
    def __init__(self, scaler_path="models/robust_scaler.pkl"):
        print("⚙️ Initializing AlphaProcessor...")
        self.device = 0 if torch.cuda.is_available() else -1
        self.model_name = "ProsusAI/finbert"
        
        # 1. Load Scaler
        try:
            self.scaler = joblib.load(scaler_path)
            print(f"✅ Scaler loaded from {scaler_path}")
        except:
            print("⚠️ Scaler not found in models/ folder.")

        # 2. Initialize FinBERT
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
        self.model = AutoModelForSequenceClassification.from_pretrained(
            self.model_name, use_safetensors=True
        )
        self.sentiment_pipe = pipeline(
            "sentiment-analysis", 
            model=self.model, 
            tokenizer=self.tokenizer, 
            device=self.device
        )

    def fetch_market_data(self, days=60):
        """
        Loads market data from your provided CSV backup.
        Bypasses Finnhub to avoid 403 errors during presentation.
        """
        print(f"📁 System: Bypassing API. Loading local market data...")
        backup_path = "data/market_data_backup.csv"
            
        if not os.path.exists(backup_path):
            print(f"🚨 FATAL: {backup_path} not found!")
            return pd.DataFrame()
                
        df = pd.read_csv(backup_path, index_col=0, parse_dates=True)
        
        # Optional: Sync dates to today for presentation realism
        # last_date = df.index[-1]
        # offset = pd.Timestamp(datetime.now().date()) - last_date
        # df.index = df.index + offset
        
        return df.tail(days)

    def process(self, df_market, df_news):
        """
        Main pipeline: News Sentiment -> Feature Engineering -> GRU Input
        """
        # 1. Process Sentiment from headlines
        df_sent, df_news_scored = self._generate_sentiment_profile(df_news)
        
        # 2. Merge and engineer all 14 features
        df_features = self._engineer_14_features(df_market, df_sent)
        
        # 3. Extract metadata for Streamlit UI
        latest_metrics = {
            "Sent_Mean": df_features['Sent_Mean'].iloc[-1],
            "News_Volume": int(np.exp(df_features['News_Volume'].iloc[-1]) - 1),
            "Panic_Interaction": df_features['Sent_x_VIX'].iloc[-1],
            "RSI": df_features['RSI'].iloc[-1] * 100
        }

        # 4. Prepare 30-day window for GRU
        final_window = df_features.tail(30).values
        scaled_window = self.scaler.transform(final_window)
        input_tensor = np.expand_dims(scaled_window, axis=0).astype('float32')

        return input_tensor, latest_metrics, df_features, df_news_scored

    def _generate_sentiment_profile(self, df_news):
        print("🧠 Running FinBERT Batch Analysis...")
        titles = df_news['Title'].astype(str).tolist()
        
        # Batch processing to handle 1700+ headlines efficiently
        results = self.sentiment_pipe(titles, batch_size=32, truncation=True)
        
        scores = []
        for res in results:
            label, score = res['label'].lower(), res['score']
            scores.append(score if label == 'positive' else -score if label == 'negative' else 0.0)
        
        df_news['Score'] = scores
        df_news['Date'] = pd.to_datetime(df_news['Date']).dt.date
        grouped = df_news.groupby('Date')['Score']
        
        daily = pd.DataFrame({
            'Sent_Mean': grouped.mean(),
            'Sent_Intensity': grouped.apply(lambda x: x.abs().mean()),
            'News_Volume': np.log1p(grouped.count()),
            'Net_Bull': grouped.apply(lambda x: x.sum() / (len(x) + 1))
        }).fillna(0.0)
        
        daily.index = pd.to_datetime(daily.index)
        return daily, df_news

    def _engineer_14_features(self, df, df_sent):
        data = df.copy()
        
        data.columns = [c.capitalize() for c in data.columns]
        if 'Vix' in data.columns: data = data.rename(columns={'Vix': 'VIX'})
        
        # --- QUANT BRANCH (7 Features) ---
        tp = (data['High'] + data['Low'] + data['Close']) / 3
        vwap = (tp * data['Volume']).rolling(20).sum() / (data['Volume'].rolling(20).sum() + 1e-9)
        data['VWAP_Dist'] = np.log(data['Close'] / vwap)
        
        delta = data['Close'].diff()
        gain = (delta.where(delta > 0, 0)).rolling(14).mean()
        loss = (-delta.where(delta < 0, 0)).rolling(14).mean()
        data['RSI'] = (100 - (100 / (1 + (gain/(loss + 1e-9))))) / 100.0
        
        ema_12 = data['Close'].ewm(span=12).mean()
        ema_26 = data['Close'].ewm(span=26).mean()
        macd = ema_12 - ema_26
        data['MACD_Hist'] = (macd - macd.ewm(span=9).mean()) / data['Close']
        
        data['VIX_Norm'] = data['VIX'] / 100.0
        data['VIX_Change'] = data['VIX'].pct_change()
        
        tr = pd.concat([data['High']-data['Low'], 
                        abs(data['High']-data['Close'].shift()), 
                        abs(data['Low']-data['Close'].shift())], axis=1).max(axis=1)
        data['ATR_Dist'] = np.tanh((data['Close'] - data['Close'].rolling(22).mean()) / (tr.rolling(14).mean() + 1e-9))
        data['Realized_Vol'] = data['Close'].pct_change().rolling(10).std() * 10

        # --- SENTIMENT BRANCH (7 Features) ---
        data.index = pd.to_datetime(data.index)
        data = data.join(df_sent, how='left').fillna(0.0)
        
        data['Sent_Mean_Delta'] = data['Sent_Mean'].diff().fillna(0.0)
        data['Sent_Mean_EMA'] = data['Sent_Mean'].ewm(span=3).mean()
        data['Sent_x_VIX'] = data['Sent_Mean'] * data['VIX_Norm']

        feature_cols = [
            'VWAP_Dist', 'RSI', 'MACD_Hist', 'VIX_Norm', 'VIX_Change', 'ATR_Dist', 'Realized_Vol',
            'Sent_Mean', 'Sent_Intensity', 'News_Volume', 'Net_Bull', 'Sent_Mean_Delta', 'Sent_Mean_EMA', 'Sent_x_VIX'
        ]
        return data[feature_cols].dropna()