#!/usr/bin/env python3
"""
Deep Analysis - Understand what's working and what's not
=========================================================
"""

import pandas as pd
import numpy as np
from datetime import datetime
from pathlib import Path
import json
import warnings
warnings.filterwarnings('ignore')

import joblib
from itertools import product

DATA_DIR = Path('/var/www/html/pippo.cuttalo.com/data')
MODEL_DIR = Path('/var/www/html/pippo.cuttalo.com/models')

def load_data():
    dfs = []
    for f in sorted(DATA_DIR.glob('prices_BTC_EUR_*.csv')):
        df = pd.read_csv(f)
        if df['timestamp'].dtype == 'int64' or str(df['timestamp'].iloc[0]).isdigit():
            ts = pd.to_numeric(df['timestamp'])
            if ts.iloc[0] > 1e12:
                df['timestamp'] = pd.to_datetime(ts, unit='ms')
            else:
                df['timestamp'] = pd.to_datetime(ts, unit='s')
        else:
            df['timestamp'] = pd.to_datetime(df['timestamp'])
        df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']]
        dfs.append(df)
    df = pd.concat(dfs, ignore_index=True)
    return df.sort_values('timestamp').drop_duplicates(subset='timestamp').reset_index(drop=True)


def analyze_price_movements(df):
    """Analyze actual price movements to understand market behavior."""
    print("\n" + "="*60)
    print("ANALISI MOVIMENTI PREZZI")
    print("="*60)

    close = df['close']

    # Analyze returns at different horizons
    for horizon in [15, 30, 60, 120, 240, 480]:
        future_return = close.shift(-horizon) / close - 1

        up_moves = (future_return > 0.01).sum()
        down_moves = (future_return < -0.01).sum()
        big_up = (future_return > 0.02).sum()
        big_down = (future_return < -0.02).sum()

        print(f"\nOrizzonte {horizon} minuti:")
        print(f"  Movimenti >1%: UP={up_moves:,} DOWN={down_moves:,}")
        print(f"  Movimenti >2%: UP={big_up:,} DOWN={big_down:,}")
        print(f"  Media return: {future_return.mean()*100:.4f}%")
        print(f"  Std return: {future_return.std()*100:.2f}%")


def test_simple_strategies(df):
    """Test simple baseline strategies to understand what works."""
    print("\n" + "="*60)
    print("TEST STRATEGIE SEMPLICI")
    print("="*60)

    close = df['close'].values
    n = len(close)

    results = []

    # Strategy 1: Buy and Hold
    buy_hold_return = (close[-1] / close[500] - 1) * 100
    results.append(('Buy & Hold', buy_hold_return, 1))

    # Strategy 2: MA Crossover
    for fast, slow in [(10, 50), (20, 100), (50, 200)]:
        ma_fast = pd.Series(close).rolling(fast).mean().values
        ma_slow = pd.Series(close).rolling(slow).mean().values

        capital = 10000
        position = 0

        for i in range(slow + 1, n):
            if position == 0 and ma_fast[i] > ma_slow[i] and ma_fast[i-1] <= ma_slow[i-1]:
                # Buy
                position = capital / close[i]
                capital = 0
            elif position > 0 and ma_fast[i] < ma_slow[i] and ma_fast[i-1] >= ma_slow[i-1]:
                # Sell
                capital = position * close[i]
                position = 0

        if position > 0:
            capital = position * close[-1]

        ret = (capital / 10000 - 1) * 100
        results.append((f'MA {fast}/{slow}', ret, 0))

    # Strategy 3: RSI
    for oversold, overbought in [(30, 70), (25, 75), (20, 80)]:
        delta = pd.Series(close).diff()
        gain = (delta.where(delta > 0, 0)).rolling(14).mean()
        loss = (-delta.where(delta < 0, 0)).rolling(14).mean()
        rs = gain / loss
        rsi = (100 - (100 / (1 + rs))).values

        capital = 10000
        position = 0

        for i in range(100, n):
            if position == 0 and rsi[i] < oversold:
                position = capital / close[i]
                capital = 0
            elif position > 0 and rsi[i] > overbought:
                capital = position * close[i]
                position = 0

        if position > 0:
            capital = position * close[-1]

        ret = (capital / 10000 - 1) * 100
        results.append((f'RSI {oversold}/{overbought}', ret, 0))

    # Strategy 4: Breakout
    for lookback in [20, 50, 100]:
        capital = 10000
        position = 0
        entry_price = 0

        high_roll = pd.Series(close).rolling(lookback).max().values
        low_roll = pd.Series(close).rolling(lookback).min().values

        for i in range(lookback + 1, n):
            if position == 0:
                if close[i] > high_roll[i-1]:
                    position = capital / close[i]
                    entry_price = close[i]
                    capital = 0
            else:
                # Exit on 2% stop loss or opposite breakout
                if close[i] < entry_price * 0.98 or close[i] < low_roll[i-1]:
                    capital = position * close[i]
                    position = 0

        if position > 0:
            capital = position * close[-1]

        ret = (capital / 10000 - 1) * 100
        results.append((f'Breakout {lookback}', ret, 0))

    print("\nRisultati strategie semplici:")
    print("-" * 40)
    for name, ret, _ in sorted(results, key=lambda x: x[1], reverse=True):
        print(f"  {name:20s}: {ret:+.2f}%")

    return results


def analyze_optimal_entries(df):
    """Find what would have been optimal entry points."""
    print("\n" + "="*60)
    print("ANALISI ENTRY OTTIMALI")
    print("="*60)

    close = df['close'].values
    n = len(close)

    # For each point, calculate max gain achievable
    optimal_trades = []

    for horizon in [30, 60, 120]:
        print(f"\nOrizzonte {horizon} minuti:")

        wins = 0
        losses = 0
        total_pnl = 0

        for i in range(500, n - horizon, horizon):
            future_prices = close[i:i+horizon]
            entry = close[i]

            max_up = (max(future_prices) / entry - 1)
            max_down = (entry / min(future_prices) - 1)

            # Simula trade con SL 2% e TP 3%
            for j, price in enumerate(future_prices):
                pnl = (price / entry - 1)
                if pnl <= -0.02:
                    total_pnl -= 0.02
                    losses += 1
                    break
                elif pnl >= 0.03:
                    total_pnl += 0.03
                    wins += 1
                    break
            else:
                # Exit at end
                total_pnl += (future_prices[-1] / entry - 1)
                if future_prices[-1] > entry:
                    wins += 1
                else:
                    losses += 1

        total = wins + losses
        print(f"  Trades: {total}, Wins: {wins}, Losses: {losses}")
        print(f"  Win Rate: {wins/total*100:.1f}%")
        print(f"  Total P&L: {total_pnl*100:.2f}%")


def find_predictable_patterns(df):
    """Find patterns that are actually predictable."""
    print("\n" + "="*60)
    print("RICERCA PATTERN PREDITTIVI")
    print("="*60)

    close = df['close']
    high = df['high']
    low = df['low']

    # Calculate indicators
    rsi = calculate_rsi(close, 14)
    ma20 = close.rolling(20).mean()
    ma50 = close.rolling(50).mean()
    atr = calculate_atr(high, low, close, 14)

    # Future returns
    future_60 = close.shift(-60) / close - 1

    # Test different conditions
    conditions = [
        ('RSI < 30', rsi < 30),
        ('RSI < 25', rsi < 25),
        ('RSI > 70', rsi > 70),
        ('RSI > 75', rsi > 75),
        ('Price > MA20 > MA50', (close > ma20) & (ma20 > ma50)),
        ('Price < MA20 < MA50', (close < ma20) & (ma20 < ma50)),
        ('Big drop (-2%)', close.pct_change(10) < -0.02),
        ('Big rally (+2%)', close.pct_change(10) > 0.02),
        ('Low volatility', atr/close < atr.rolling(100).mean()/close.rolling(100).mean() * 0.7),
        ('High volatility', atr/close > atr.rolling(100).mean()/close.rolling(100).mean() * 1.3),
    ]

    print("\nAnalisi condizioni -> Return 60min futuro:")
    print("-" * 60)

    for name, condition in conditions:
        mask = condition & future_60.notna()
        if mask.sum() > 100:
            avg_ret = future_60[mask].mean() * 100
            std_ret = future_60[mask].std() * 100
            win_rate = (future_60[mask] > 0).mean() * 100
            count = mask.sum()

            print(f"  {name:30s}: avg={avg_ret:+.3f}%, win={win_rate:.1f}%, n={count:,}")


def calculate_rsi(prices, period=14):
    delta = prices.diff()
    gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
    loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
    rs = gain / loss
    return 100 - (100 / (1 + rs))


def calculate_atr(high, low, close, period=14):
    tr1 = high - low
    tr2 = abs(high - close.shift(1))
    tr3 = abs(low - close.shift(1))
    tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
    return tr.rolling(window=period).mean()


def test_mean_reversion(df):
    """Test mean reversion strategies - often work better in crypto."""
    print("\n" + "="*60)
    print("TEST MEAN REVERSION")
    print("="*60)

    close = df['close'].values
    n = len(close)

    # Bollinger Band mean reversion
    for bb_std in [1.5, 2.0, 2.5]:
        sma = pd.Series(close).rolling(20).mean().values
        std = pd.Series(close).rolling(20).std().values
        upper = sma + bb_std * std
        lower = sma - bb_std * std

        capital = 10000
        position = 0
        entry_price = 0
        trades = []

        for i in range(100, n):
            if position == 0:
                # Buy at lower band
                if close[i] < lower[i]:
                    position = 1
                    entry_price = close[i]
                    trade_capital = capital * 0.3
                    capital -= trade_capital
                    position_size = trade_capital / close[i]
                # Short at upper band
                elif close[i] > upper[i]:
                    position = -1
                    entry_price = close[i]
                    trade_capital = capital * 0.3
                    capital -= trade_capital
                    position_size = trade_capital / close[i]
            else:
                # Exit at middle or stop loss
                if position == 1:
                    pnl_pct = (close[i] - entry_price) / entry_price
                    if close[i] > sma[i] or pnl_pct < -0.02 or pnl_pct > 0.03:
                        pnl = position_size * (close[i] - entry_price)
                        capital += trade_capital + pnl
                        trades.append(pnl)
                        position = 0
                else:  # Short
                    pnl_pct = (entry_price - close[i]) / entry_price
                    if close[i] < sma[i] or pnl_pct < -0.02 or pnl_pct > 0.03:
                        pnl = position_size * (entry_price - close[i])
                        capital += trade_capital + pnl
                        trades.append(pnl)
                        position = 0

        if position != 0:
            if position == 1:
                pnl = position_size * (close[-1] - entry_price)
            else:
                pnl = position_size * (entry_price - close[-1])
            capital += trade_capital + pnl
            trades.append(pnl)

        ret = (capital / 10000 - 1) * 100
        wins = sum(1 for t in trades if t > 0)
        total = len(trades)
        wr = wins/total*100 if total > 0 else 0

        print(f"BB Mean Reversion (std={bb_std}): Return={ret:+.2f}%, Trades={total}, WR={wr:.1f}%")


def test_trend_following(df):
    """Test trend following with proper filters."""
    print("\n" + "="*60)
    print("TEST TREND FOLLOWING")
    print("="*60)

    close = df['close'].values
    high = df['high'].values
    low = df['low'].values
    n = len(close)

    close_s = pd.Series(close)
    high_s = pd.Series(high)
    low_s = pd.Series(low)

    # ADX filter for trend strength
    tr = pd.concat([
        high_s - low_s,
        abs(high_s - close_s.shift(1)),
        abs(low_s - close_s.shift(1))
    ], axis=1).max(axis=1)

    plus_dm = high_s.diff()
    minus_dm = low_s.diff()
    plus_dm[plus_dm < 0] = 0
    minus_dm[minus_dm > 0] = 0

    tr_sum = tr.rolling(14).sum()
    plus_di = 100 * (plus_dm.rolling(14).sum() / tr_sum)
    minus_di = 100 * (abs(minus_dm.rolling(14).sum()) / tr_sum)
    dx = 100 * abs(plus_di - minus_di) / (plus_di + minus_di + 0.0001)
    adx = dx.rolling(14).mean().values

    ma_fast = close_s.rolling(20).mean().values
    ma_slow = close_s.rolling(50).mean().values

    for adx_threshold in [20, 25, 30]:
        capital = 10000
        position = 0
        entry_price = 0
        trades = []

        for i in range(100, n):
            if position == 0:
                # Only trade when ADX shows trend
                if adx[i] > adx_threshold:
                    if ma_fast[i] > ma_slow[i] and ma_fast[i-1] <= ma_slow[i-1]:
                        position = 1
                        entry_price = close[i]
                        trade_capital = capital * 0.3
                        capital -= trade_capital
                        position_size = trade_capital / close[i]
                    elif ma_fast[i] < ma_slow[i] and ma_fast[i-1] >= ma_slow[i-1]:
                        position = -1
                        entry_price = close[i]
                        trade_capital = capital * 0.3
                        capital -= trade_capital
                        position_size = trade_capital / close[i]
            else:
                if position == 1:
                    pnl_pct = (close[i] - entry_price) / entry_price
                    # Trail stop or reverse signal
                    if pnl_pct < -0.015 or (ma_fast[i] < ma_slow[i]):
                        pnl = position_size * (close[i] - entry_price)
                        capital += trade_capital + pnl
                        trades.append(pnl)
                        position = 0
                else:
                    pnl_pct = (entry_price - close[i]) / entry_price
                    if pnl_pct < -0.015 or (ma_fast[i] > ma_slow[i]):
                        pnl = position_size * (entry_price - close[i])
                        capital += trade_capital + pnl
                        trades.append(pnl)
                        position = 0

        if position != 0:
            if position == 1:
                pnl = position_size * (close[-1] - entry_price)
            else:
                pnl = position_size * (entry_price - close[-1])
            capital += trade_capital + pnl
            trades.append(pnl)

        ret = (capital / 10000 - 1) * 100
        wins = sum(1 for t in trades if t > 0)
        total = len(trades)
        wr = wins/total*100 if total > 0 else 0

        print(f"Trend + ADX>{adx_threshold}: Return={ret:+.2f}%, Trades={total}, WR={wr:.1f}%")


def main():
    print("="*60)
    print("DEEP ANALYSIS")
    print("="*60)

    df = load_data()
    print(f"\nDati: {len(df):,} candele")
    print(f"Periodo: {df['timestamp'].min()} - {df['timestamp'].max()}")

    # Various analyses
    analyze_price_movements(df)
    test_simple_strategies(df)
    analyze_optimal_entries(df)
    find_predictable_patterns(df)
    test_mean_reversion(df)
    test_trend_following(df)

    print("\n" + "="*60)
    print("ANALISI COMPLETATA")
    print("="*60)


if __name__ == '__main__':
    main()