#!/usr/bin/env python3
"""
Configuration Optimizer - Grid Search for Best Trading Parameters
=================================================================
Tests all combinations of trading parameters on historical data.
"""

import pandas as pd
import numpy as np
from datetime import datetime, timedelta
from pathlib import Path
from itertools import product
import json
import warnings
warnings.filterwarnings('ignore')

import joblib

# Configuration
DATA_DIR = Path('/var/www/html/pippo.cuttalo.com/data')
MODEL_DIR = Path('/var/www/html/pippo.cuttalo.com/models')
RESULTS_DIR = Path('/var/www/html/pippo.cuttalo.com/optimization_results')
RESULTS_DIR.mkdir(exist_ok=True)

# Parameter grid for optimization
PARAM_GRID = {
    'stop_loss': [0.01, 0.015, 0.02, 0.025, 0.03],  # 1% to 3%
    'take_profit': [0.02, 0.03, 0.04, 0.05, 0.06],  # 2% to 6%
    'position_size': [0.1, 0.2, 0.3, 0.5],  # 10% to 50% of capital
    'confidence_threshold': [0.5, 0.55, 0.6, 0.65, 0.7],  # Min confidence for trade
    'fee_rate': [0.001],  # 0.1% fee (Binance typical)
}

# Backtest settings
INITIAL_CAPITAL = 10000
SLIPPAGE = 0.0003  # 0.03%


class Backtester:
    """Backtesting engine for trading strategies."""

    def __init__(self, df, model, feature_names, params):
        self.df = df
        self.model = model
        self.feature_names = feature_names
        self.params = params

        self.capital = INITIAL_CAPITAL
        self.position = 0  # 0=flat, 1=long, -1=short
        self.position_amount = 0
        self.entry_price = 0
        self.entry_time = None

        self.trades = []
        self.equity_curve = []

    def calculate_features(self, idx, window=200):
        """Calculate features for prediction at given index."""
        if idx < window:
            return None

        # Get window of data
        data = self.df.iloc[idx - window:idx + 1].copy()

        # Import feature creation from train_model
        from train_model import create_features

        features = create_features(data)

        if features.iloc[-1].isna().any():
            return None

        # Return only the features we need
        return features[self.feature_names].iloc[-1:].values

    def get_signal(self, features):
        """Get trading signal from model."""
        if features is None:
            return 0, 0

        proba = self.model.predict(features)[0]
        # proba is [short, neutral, long]

        long_prob = proba[2]
        short_prob = proba[0]

        if long_prob > self.params['confidence_threshold'] and long_prob > short_prob:
            return 1, long_prob
        elif short_prob > self.params['confidence_threshold'] and short_prob > long_prob:
            return -1, short_prob

        return 0, max(proba)

    def execute_entry(self, idx, direction, confidence):
        """Execute entry trade."""
        price = self.df.iloc[idx]['close']

        trade_value = self.capital * self.params['position_size']
        fee = trade_value * self.params['fee_rate']

        # Apply slippage
        if direction == 1:
            exec_price = price * (1 + SLIPPAGE)
        else:
            exec_price = price * (1 - SLIPPAGE)

        amount = (trade_value - fee) / exec_price

        self.capital -= trade_value
        self.position = direction
        self.position_amount = amount
        self.entry_price = exec_price
        self.entry_time = self.df.iloc[idx]['timestamp']

        return {
            'type': 'entry',
            'direction': 'long' if direction == 1 else 'short',
            'price': exec_price,
            'amount': amount,
            'fee': fee,
            'confidence': confidence,
            'time': self.entry_time
        }

    def execute_exit(self, idx, reason):
        """Execute exit trade."""
        price = self.df.iloc[idx]['close']

        # Apply slippage
        if self.position == 1:
            exec_price = price * (1 - SLIPPAGE)
            gross_pnl = self.position_amount * (exec_price - self.entry_price)
        else:
            exec_price = price * (1 + SLIPPAGE)
            gross_pnl = self.position_amount * (self.entry_price - exec_price)

        fee = self.position_amount * exec_price * self.params['fee_rate']
        net_pnl = gross_pnl - fee
        pnl_pct = net_pnl / (self.position_amount * self.entry_price)

        self.capital += self.position_amount * self.entry_price + net_pnl

        trade = {
            'type': 'exit',
            'direction': 'long' if self.position == 1 else 'short',
            'entry_price': self.entry_price,
            'exit_price': exec_price,
            'amount': self.position_amount,
            'gross_pnl': gross_pnl,
            'net_pnl': net_pnl,
            'pnl_pct': pnl_pct,
            'fee': fee,
            'reason': reason,
            'entry_time': self.entry_time,
            'exit_time': self.df.iloc[idx]['timestamp']
        }

        self.position = 0
        self.position_amount = 0
        self.entry_price = 0
        self.entry_time = None

        return trade

    def check_exit_conditions(self, idx):
        """Check if exit conditions are met."""
        if self.position == 0:
            return None

        price = self.df.iloc[idx]['close']

        # Calculate P&L
        if self.position == 1:
            pnl_pct = (price - self.entry_price) / self.entry_price
        else:
            pnl_pct = (self.entry_price - price) / self.entry_price

        # Check stop loss
        if pnl_pct <= -self.params['stop_loss']:
            return 'stop_loss'

        # Check take profit
        if pnl_pct >= self.params['take_profit']:
            return 'take_profit'

        return None

    def run(self, start_idx=500, progress_interval=10000):
        """Run backtest."""
        total = len(self.df) - start_idx

        for i, idx in enumerate(range(start_idx, len(self.df))):
            # Record equity
            equity = self.capital
            if self.position != 0:
                price = self.df.iloc[idx]['close']
                if self.position == 1:
                    equity += self.position_amount * price
                else:
                    unrealized = (self.entry_price - price) * self.position_amount
                    equity += self.position_amount * self.entry_price + unrealized

            self.equity_curve.append({
                'timestamp': self.df.iloc[idx]['timestamp'],
                'equity': equity,
                'position': self.position
            })

            # Check exit conditions first
            exit_reason = self.check_exit_conditions(idx)
            if exit_reason:
                trade = self.execute_exit(idx, exit_reason)
                self.trades.append(trade)
                continue

            # Get features and signal
            if self.position == 0:
                features = self.calculate_features(idx)
                signal, confidence = self.get_signal(features)

                if signal != 0:
                    entry = self.execute_entry(idx, signal, confidence)
                    self.trades.append(entry)

        # Close any open position at the end
        if self.position != 0:
            trade = self.execute_exit(len(self.df) - 1, 'end_of_test')
            self.trades.append(trade)

        return self.calculate_metrics()

    def calculate_metrics(self):
        """Calculate performance metrics."""
        if not self.equity_curve:
            return None

        equity_df = pd.DataFrame(self.equity_curve)

        # Filter only completed trades (exits)
        completed_trades = [t for t in self.trades if t['type'] == 'exit']

        if not completed_trades:
            return {
                'total_return': 0,
                'total_return_pct': 0,
                'num_trades': 0,
                'win_rate': 0,
                'sharpe_ratio': 0,
                'max_drawdown': 0,
                'profit_factor': 0,
                'avg_trade_pnl': 0,
                'avg_win': 0,
                'avg_loss': 0,
            }

        # Calculate metrics
        final_equity = equity_df['equity'].iloc[-1]
        total_return = final_equity - INITIAL_CAPITAL
        total_return_pct = total_return / INITIAL_CAPITAL

        wins = [t for t in completed_trades if t['net_pnl'] > 0]
        losses = [t for t in completed_trades if t['net_pnl'] <= 0]

        win_rate = len(wins) / len(completed_trades) if completed_trades else 0

        # Sharpe ratio (annualized, assuming 1-min data)
        returns = equity_df['equity'].pct_change().dropna()
        if len(returns) > 0 and returns.std() > 0:
            sharpe = returns.mean() / returns.std() * np.sqrt(525600)  # 525600 minutes/year
        else:
            sharpe = 0

        # Max drawdown
        cummax = equity_df['equity'].cummax()
        drawdown = (equity_df['equity'] - cummax) / cummax
        max_drawdown = abs(drawdown.min())

        # Profit factor
        gross_profit = sum(t['net_pnl'] for t in wins) if wins else 0
        gross_loss = abs(sum(t['net_pnl'] for t in losses)) if losses else 1
        profit_factor = gross_profit / gross_loss if gross_loss > 0 else gross_profit

        # Average trade metrics
        avg_trade_pnl = np.mean([t['net_pnl'] for t in completed_trades])
        avg_win = np.mean([t['net_pnl'] for t in wins]) if wins else 0
        avg_loss = np.mean([t['net_pnl'] for t in losses]) if losses else 0

        # By reason
        by_reason = {}
        for t in completed_trades:
            reason = t['reason']
            if reason not in by_reason:
                by_reason[reason] = {'count': 0, 'pnl': 0}
            by_reason[reason]['count'] += 1
            by_reason[reason]['pnl'] += t['net_pnl']

        return {
            'total_return': total_return,
            'total_return_pct': total_return_pct,
            'final_equity': final_equity,
            'num_trades': len(completed_trades),
            'win_rate': win_rate,
            'sharpe_ratio': sharpe,
            'max_drawdown': max_drawdown,
            'profit_factor': profit_factor,
            'avg_trade_pnl': avg_trade_pnl,
            'avg_win': avg_win,
            'avg_loss': avg_loss,
            'wins': len(wins),
            'losses': len(losses),
            'by_reason': by_reason
        }


def load_model():
    """Load the latest trained model."""
    latest_path = MODEL_DIR / 'latest'
    if not latest_path.exists():
        raise FileNotFoundError("No trained model found. Run train_model.py first.")

    model_path = latest_path.resolve()
    model = joblib.load(model_path / 'model.joblib')

    with open(model_path / 'features.json') as f:
        feature_names = json.load(f)

    print(f"Loaded model from {model_path}")
    return model, feature_names


def load_data():
    """Load price data."""
    print("Loading price data...")

    dfs = []
    for f in sorted(DATA_DIR.glob('prices_BTC_EUR_*.csv')):
        df = pd.read_csv(f)

        # Handle different timestamp formats
        if df['timestamp'].dtype == 'int64' or (df['timestamp'].dtype == 'object' and str(df['timestamp'].iloc[0]).isdigit()):
            # Unix timestamp (seconds or milliseconds)
            ts = pd.to_numeric(df['timestamp'])
            if ts.iloc[0] > 1e12:  # Milliseconds
                df['timestamp'] = pd.to_datetime(ts, unit='ms')
            else:  # Seconds
                df['timestamp'] = pd.to_datetime(ts, unit='s')
        else:
            # String datetime
            df['timestamp'] = pd.to_datetime(df['timestamp'])

        # Keep only necessary columns
        df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']]
        dfs.append(df)

    df = pd.concat(dfs, ignore_index=True)
    df = df.sort_values('timestamp').drop_duplicates(subset='timestamp').reset_index(drop=True)

    print(f"Loaded {len(df):,} candles")
    return df


def run_optimization(df, model, feature_names, sample_size=None):
    """Run grid search optimization."""
    print("\n" + "=" * 60)
    print("CONFIGURATION OPTIMIZATION")
    print("=" * 60)

    # Sample data if specified (for faster testing)
    if sample_size and len(df) > sample_size:
        df = df.iloc[-sample_size:].reset_index(drop=True)
        print(f"Using last {sample_size:,} candles for optimization")

    # Generate all parameter combinations
    param_names = list(PARAM_GRID.keys())
    param_values = list(PARAM_GRID.values())
    combinations = list(product(*param_values))

    print(f"\nTesting {len(combinations)} parameter combinations...")
    print(f"Parameters: {param_names}")

    results = []

    for i, combo in enumerate(combinations):
        params = dict(zip(param_names, combo))

        print(f"\n[{i+1}/{len(combinations)}] Testing: SL={params['stop_loss']:.1%}, "
              f"TP={params['take_profit']:.1%}, Size={params['position_size']:.0%}, "
              f"Conf={params['confidence_threshold']:.0%}")

        try:
            backtester = Backtester(df, model, feature_names, params)
            metrics = backtester.run(start_idx=500)

            if metrics:
                result = {**params, **metrics}
                results.append(result)

                print(f"  Return: {metrics['total_return_pct']*100:+.2f}% | "
                      f"Trades: {metrics['num_trades']} | "
                      f"Win Rate: {metrics['win_rate']*100:.1f}% | "
                      f"Sharpe: {metrics['sharpe_ratio']:.2f} | "
                      f"Max DD: {metrics['max_drawdown']*100:.1f}%")

        except Exception as e:
            print(f"  Error: {e}")

    return results


def analyze_results(results):
    """Analyze and rank results."""
    if not results:
        print("No results to analyze")
        return None

    df_results = pd.DataFrame(results)

    # Score each configuration (weighted combination of metrics)
    df_results['score'] = (
        df_results['total_return_pct'] * 0.3 +
        df_results['sharpe_ratio'] * 0.25 +
        df_results['win_rate'] * 0.2 +
        df_results['profit_factor'].clip(upper=5) * 0.15 -
        df_results['max_drawdown'] * 0.1
    )

    df_results = df_results.sort_values('score', ascending=False)

    print("\n" + "=" * 60)
    print("TOP 10 CONFIGURATIONS")
    print("=" * 60)

    for i, row in df_results.head(10).iterrows():
        print(f"\n#{df_results.index.get_loc(i) + 1} Score: {row['score']:.4f}")
        print(f"  SL: {row['stop_loss']:.1%} | TP: {row['take_profit']:.1%} | "
              f"Size: {row['position_size']:.0%} | Conf: {row['confidence_threshold']:.0%}")
        print(f"  Return: {row['total_return_pct']*100:+.2f}% | Trades: {row['num_trades']:.0f} | "
              f"Win Rate: {row['win_rate']*100:.1f}%")
        print(f"  Sharpe: {row['sharpe_ratio']:.2f} | Max DD: {row['max_drawdown']*100:.1f}% | "
              f"PF: {row['profit_factor']:.2f}")

    return df_results


def save_results(df_results, best_config):
    """Save optimization results."""
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')

    # Save all results
    results_file = RESULTS_DIR / f'optimization_{timestamp}.csv'
    df_results.to_csv(results_file, index=False)
    print(f"\nResults saved to {results_file}")

    # Save best configuration
    best_config_file = RESULTS_DIR / 'best_config.json'
    with open(best_config_file, 'w') as f:
        json.dump(best_config, f, indent=2)
    print(f"Best config saved to {best_config_file}")

    return results_file


def main():
    # Load model
    model, feature_names = load_model()

    # Load data
    df = load_data()

    # Run optimization (use sample for faster testing, remove for full)
    results = run_optimization(df, model, feature_names, sample_size=200000)

    # Analyze results
    df_results = analyze_results(results)

    if df_results is not None and len(df_results) > 0:
        # Get best configuration
        best = df_results.iloc[0]
        best_config = {
            'stop_loss': float(best['stop_loss']),
            'take_profit': float(best['take_profit']),
            'position_size': float(best['position_size']),
            'confidence_threshold': float(best['confidence_threshold']),
            'fee_rate': float(best['fee_rate']),
            'metrics': {
                'total_return_pct': float(best['total_return_pct']),
                'win_rate': float(best['win_rate']),
                'sharpe_ratio': float(best['sharpe_ratio']),
                'max_drawdown': float(best['max_drawdown']),
                'profit_factor': float(best['profit_factor']),
                'num_trades': int(best['num_trades']),
            }
        }

        save_results(df_results, best_config)

        print("\n" + "=" * 60)
        print("BEST CONFIGURATION")
        print("=" * 60)
        print(json.dumps(best_config, indent=2))

    print("\n" + "=" * 60)
    print("OPTIMIZATION COMPLETE")
    print("=" * 60)


if __name__ == '__main__':
    main()