
    ri9                        d Z ddlZddlZddlmZmZ ddlmZ ddl	Z	ddl
Z
 e
j        d           ddlZddlmZ ddlmZmZmZmZ ddlZ ed          Z ed	          Ze                    d
           g dZdZdZdZdZdZdZd Z d$dZ!d%dZ"d&dZ#d$dZ$d Z%d$dZ&d'dZ'd(dZ(d Z)d Z*d)d!Z+d" Z,e-d#k    r e,            \  Z.Z/Z0dS dS )*z
V9 Model Training - Complete Pipeline
=====================================
Trains the trading model on all available historical data.
    N)datetime	timedelta)Pathignore)TimeSeriesSplit)accuracy_scoreprecision_scorerecall_scoref1_scorez$/var/www/html/pippo.cuttalo.com/dataz&/var/www/html/pippo.cuttalo.com/modelsTexist_ok)   
      2   d               	   r      c                     t          d           g } t          t                              d                    D ]}t          d|j         d           t          j        |          }|d         j        dk    s6|d         j        dk    r|d         j        d         	                                r_t          j
        |d                   }|j        d         d	k    rt          j        |d
          |d<   n7t          j        |d          |d<   nt          j        |d                   |d<   |g d         }|                     |           t          j        | d          }|                    d                              d                              d          }t          dt#          |          d           t          d|d                                          d|d                                                     |S )z Load and combine all price data.zLoading price data...zprices_BTC_EUR_*.csvz
  Loading z...	timestampint64objectr   g   mBms)units)r   openhighlowclosevolumeT)ignore_index)subset)dropzTotal candles: ,zDate range: z to )printsortedDATA_DIRglobnamepdread_csvdtypeilocisdigit
to_numericto_datetimeappendconcatsort_valuesdrop_duplicatesreset_indexlenminmax)dfsfdftss       ?/var/www/html/pippo.cuttalo.com/services/trading/train_model.py	load_datarB   %   s   	
!"""
CHMM"899::  &16&&&'''[^^ k? G++;0E0Q0QVXYdVeVjklVmVuVuVwVw0Qr+//BwqzD  "$.$"?"?"?;"$.#">">">; !nR_==B{O GGGH

2	3T	*	*	*B		$	$	4	4K	4	H	H	T	TZ^	T	_	_B	
'CGG
'
'
'(((	
KK,,..
K
KB{O4G4G4I4I
K
KLLLI    c                 L   |                                  }|                    |dk    d                              |                                          }|                    |dk     d                               |                                          }||z  }ddd|z   z  z
  S )zCalculate RSI indicator.r   windowr      )diffwhererollingmean)pricesperioddeltagainlossrss         rA   calculate_rsirR   G   s    KKMMEKK	1%%..f.==BBDDD[[A&&&//v/>>CCEED	B#R.!!rC   c                    |                      |d                                          }|                      |d                                          }||z
  }|                     |d                                          }||z
  }|||fS )zCalculate MACD indicator.Fspanadjust)ewmrK   )	rL   fastslowsignalexp1exp2macdmacd_signal	macd_hists	            rA   calculate_macdr`   P   s    ::4:..3355D::4:..3355D$;D((u(55::<<K{"Ii''rC   c                     |                      |                                          }|                      |                                          }|||z  z   }|||z  z
  }|||fS )zCalculate Bollinger Bands.rE   )rJ   rK   std)rL   rM   rb   smastd_devupperlowers          rA   calculate_bollinger_bandsrg   Z   si    
...
'
'
,
,
.
.CnnFn++//11G7S=!E7S=!E#urC   c                 J   | |z
  }t          | |                    d          z
            }t          ||                    d          z
            }t          j        |||gd                              d          }|                    |                                          S )zCalculate Average True Range.rG   axisrE   )absshiftr.   r6   r<   rJ   rK   )r!   r"   r#   rM   tr1tr2tr3trs           rA   calculate_atrrq   c   s    
*C
dU[[^^#
$
$C
cEKKNN"
#
#C	Cc?	+	+	+	/	/Q	/	7	7B::V:$$))+++rC   c                 
   t          d           | d         }| d         }| d         }| d         }t          j        | j                  }t          D ]}}|                    |          |d| <   |                                                    |                                          |d| <   ||                    |          z  d	z
  |d
| <   ~dD ]u}|                    |          	                                |d| <   |
                    |d          	                                |d| <   ||d|          z  d	z
  |d| <   v|d         |d         k                        t                    |d<   |d         |d         k                        t                    |d<   |d         |d         k                        t                    |d<   t          |t                    |d<   |d         dk                         t                    |d<   |d         dk                        t                    |d<   t          |t           t"          t$                    \  }}}	||d<   ||d<   |	|d <   ||k    |                    d	          |                    d	          k    z                      t                    |d!<   ||k     |                    d	          |                    d	          k    z                      t                    |d"<   t'          |t(          t*                    \  }
}}||z
  |
|z
  z  |d#<   |
|z
  |z  |d$<   ||
z  d	z
  |d%<   ||z  d	z
  |d&<   t-          |||d'          |d(<   |d(         |z  |d)<   |                    d*          	                                |d+<   ||d+         z  |d,<   |                                |d-<   ||                    d	          k                        t                    |d.<   ||                    d	          k                         t                    |d/<   t/          || d0         z
            ||z
  d1z   z  |d2<   |t          j        || d0         gd	3                              d	3          z
  ||z
  d1z   z  |d4<   t          j        || d0         gd	3                              d	3          |z
  ||z
  d1z   z  |d5<   t7          |||d'          |d6<   | d7         j        j        |d8<   | d7         j        j        |d9<   |d9         d:k                        t                    |d;<   d<D ]#}||j        v r|                     |d	d=>           $t          d?tC          |j                   d@           |S )Az"Create all features for the model.zCreating features...r#   r!   r"   r$   indexreturn_volatility_rG   	momentum_)r   r   r   r   r   sma_FrT   ema_price_vs_sma_sma_10sma_50sma_10_50_crosssma_20sma_100sma_20_100_crosssma_200sma_50_200_crossrsi   rsi_oversoldF   rsi_overboughtr]   r^   r_   macd_cross_upmacd_cross_downbb_positionbb_widthprice_vs_bb_upperprice_vs_bb_lowerr   atratr_pctr   volume_sma_20volume_ratiovolume_changehigher_high	lower_lowr    -C6?	body_sizeri   
upper_wick
lower_wickadxr   hourday_of_weekr   
is_weekend)r{   r~   r|   r   r   ema_10ema_20ema_50ema_100ema_200r   T)rj   inplacezCreated z	 features)"r)   r.   	DataFramert   LOOKBACK_PERIODS
pct_changerJ   rb   rl   rK   rW   astypeintrR   
RSI_PERIODr`   	MACD_FAST	MACD_SLOWMACD_SIGNALrg   	BB_PERIODBB_STDrq   rk   r6   r<   r;   calculate_adxdtr   	dayofweekcolumnsr'   r:   )r?   r#   r!   r"   r$   featuresrM   r]   r^   r_   bb_upperbb_midbb_lowercols                 rA   create_featuresr   l   s   	
 !!!wKEf:D
U)C\F|"(+++H # I I','7'7'?'?#6##$+0+;+;+=+=+E+Ef+M+M+Q+Q+S+S'v''().V1D1D)Dq)H%V%%&& ) S S$)MM&$9$9$>$>$@$@!$)II6%I$H$H$M$M$O$O!-2XoVoo5N-NQR-R)))** $,H#58J#J"R"RSV"W"WH$,X$6)9L$L#T#TUX#Y#YH $,X$6)9L$L#T#TUX#Y#YH  $E:66HUO (" 4<<SAAH^"*5/B"6!>!>s!C!CH $2%I{#[#[ D+yHV)H]%H["&"4A+J[J[\]J^J^9^!_ g ghk l lH_$(;$64::a==KL]L]^_L`L`;`#a"i"ijm"n"nH ";5)V!T!THfh$x/Hx4GHH]$x/69HZ$)H$4q$8H !$)H$4q$8H ! $D#ub99HUO"5/E1HY !'r 2 2 7 7 9 9H_%(AAH^ & 1 1 3 3H_  $djjmm3;;C@@H] 399Q<</77<<H[6
 233tczF7JKH["RYr&z/B%K%K%K%O%OUV%O%W%WW\`cf\fio\opH\ i6
(;!DDDHHaHPPSVV[_be[ehn[noH\ $D#ub99HUO +).HV o0:H]&}5:BB3GGH\! 5 5 ("""MM#AtM444	
5S)**
5
5
5666OrC   c                    |                                  }|                                 }d||dk     <   d||dk    <   t          | ||d          |z  }d|                    |                                          |z  z  }dt	          |                    |                                                    |z  z  }dt	          ||z
            z  ||z   dz   z  }	|	                    |                                          }
|
S )z*Calculate ADX (Average Directional Index).r   rG   r   r   )rH   rq   rJ   sumrk   rK   )r!   r"   r#   rM   plus_dmminus_dmrp   plus_diminus_didxr   s              rA   r   r      s    iikkGxxzzHGGaKHX\	tS%	+	+f	4BW__V,,0022R78Gc(**622668899B>?H	s7X%&&	&'H*<v*E	FB
**V


!
!
#
#CJrC   <   ~jth?c                 h   t          d| d|dz  dd           | d         }|                    |                                                              |           }|                    |                                                              |           }||z  dz
  }||z  dz
  }t          j        d| j        	          }d|||k    ||k    z  <   d
|||k    ||k    z  <   t          dt          |dk              ddt          |d
k              ddt          |dk              d           |S )z
    Create labels for training.
    1 = Long opportunity (price will go up by threshold)
    -1 = Short opportunity (price will go down by threshold)
    0 = No clear signal
    zCreating labels (future=zm, threshold=r   z.1fz%)...r#   rG   r   rs   zLabels distribution: Long=r(   z, Short=z
, Neutral=)	r)   rJ   r<   rl   r;   r.   Seriesrt   r   )	r?   future_periods	thresholdr#   
future_max
future_min	future_upfuture_downlabelss	            rA   create_labelsr      sY    

Z^
Z
Z)C-
Z
Z
Z
Z[[[wKE ~..2244::N?KKJ~..2244::N?KKJe#a'I:%)KYq)))F DEFI"y;'>?@ FHFK9$y)@AB	
ps619~~
p
p
pVRZ
p
p
p^abhjkbk^l^l
p
p
pqqqMrC   c                    t          d           |ddddddd	d	d
dddd}t          d
          }g }g }t          |                    |                     D ]D\  }\  }}t          d|dz    d           | j        |         | j        |         }
}	|j        |         |j        |         }}|dz   }|dz   }t          j        |	|          }t          j        |
||          }t          j        |||gt          j        d          t          j	        d          g          }|
                    |
          }t          j        |d          dz
  }t          ||          }|                    |           |                    |           t          d|d           Ft          dt          j        |          ddt          j        |          dd           |d         |fS )z7Train LightGBM model with time series cross-validation.z
Training model...N
multiclass   multi_loglossgbdt?   g?皙?r   r   i  r   )	objective	num_classmetricboosting_type
num_leaveslearning_ratefeature_fractionbagging_fractionbagging_freqverbosen_estimatorsearly_stopping_rounds)n_splitsz  Fold rG   z/5...)label)r   	referencer   )
valid_sets	callbacksri   z    Accuracy: .4fz
Mean accuracy: z (+/- ))r)   r   	enumeratesplitr1   lgbDatasettrainearly_stoppinglog_evaluationpredictnpargmaxr   r5   rK   rb   )Xyparamstscvscoresmodelsfold	train_idxval_idxX_trainX_valy_trainy_valy_train_conv
y_val_conv
train_dataval_datamodely_pred_probay_predaccs                        rA   train_modelr     s   	
   ~%%#! # #%'
 
  A&&&DFF&/

1&>&> * *""y''q'''(((	*AF7O	*AF7O {QY
[===
;uJ*MMM	 z)"--s/A!/D/DE	
 
 
 }}U++<a00014UF++ce(s((())))	
Nbgfoo
N
N
N
N
N
N
NOOO ":vrC   c                 z   |                      |          }t          j        |d          dz
  }t          ||          }|i d}dD ]\  }}||k    }	|	                                dk    r||k    }
||k    ||k    z                                  }||k    ||k    z                                  }||k    ||k    z                                  }||z   dk    r|||z   z  nd}||z   dk    r|||z   z  nd}||z   dk    rd|z  |z  ||z   z  nd}|||t          |	                                          d|d         |<   |S )	zEvaluate model performance.rG   ri   )accuracy	per_class))r   short)r   neutral)rG   longr   r   )	precisionrecallf1supportr  )r   r   r   r   r   r   )r  X_testy_testr  r  r  resultsr   r-   mask	pred_masktpfpfnr  r  r  s                    rA   evaluate_modelr  /  s   ==((LY|!,,,q0F 
(
(C  G
 D  t88::>>%IU?v7<<>>BU?v7<<>>BU?v7<<>>B+-7a--b2gQI(*R1}}R27^^!FCLvCUYZBZBZY'9v+=>>`aB ' txxzz??	* *GK & NrC   c                     |                      d          }t          j        ||d                              dd          }|S )z"Get feature importance from model.rO   )importance_type)feature
importancer  F)	ascending)feature_importancer.   r   r7   )r  feature_namesr  importance_dfs       rA   get_feature_importancer!  S  sW    ))&)AAJL  " "   {<5{11 
 rC   v9c                    t          j                                        d          }t          d| d| z  }|                    d           t          j        | |dz             t          |dz  d          5 }t          j        t          |j
                  |           d	d	d	           n# 1 swxY w Y   t          |d
z  d          5 }t          j        ||d           d	d	d	           n# 1 swxY w Y   t          dz  }|                                r|                                 |                    |j                   t          d|            |S )zSave model and metadata.z%Y%m%d_%H%M%Smodel__Tr   zmodel.joblibzfeatures.jsonwNzmetrics.jsonr   )indentlatestz
Model saved to )r   nowstrftime	MODEL_DIRmkdirjoblibdumpr    jsonlistr   existsunlink
symlink_tor-   r)   )r  r   metricsversionr   
model_pathr>   latest_paths           rA   
save_modelr8  ^  s   ''88I;g;;	;;;Jd### KzN2333 
j?*C	0	0 -A	$x'((!,,,- - - - - - - - - - - - - - - 
j>)3	/	/ (1	'1Q''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( h&K :?+++	
*j
*
*+++s$   8(B,,B03B0
C..C25C2c                     t          d           t          d           t          d           t                       t                      } t          |           }t          | dd          }|                                                    d          |                                z  }||         }||         }| |         } t          dt          |          d	           t          t          |          d
z            }|j        d |         }|j        d |         }|j        |d          }|j        |d          }t          dt          |          d	d           t          dt          |          d	d           t          ||          \  }	}
t          d           t          |	||          }t          d           t          d|d         d           |d                                         D ]\  }}t          d|                                 d           t          d|d         d           t          d|d         d           t          d|d         d           t          d|d         d	           t          d           t          |	|j                  }|                    d                                          D ]'\  }}t          d|d           d!|d"         d#           (|
|d$<   t#          t%          j        |
                    |d%<   t#          t%          j        |
                    |d&<   t          |          |d'<   t          |          |d(<   t          |j                  |d)<   t+          |	||          }t          d*           t          d+           t          d           |	|j        |fS ),Nz<============================================================zV9 MODEL TRAININGr   r   )r   r   rG   ri   z
Valid samples: r(   r   zTrain: z sampleszTest: z
Evaluating on test set...z
Test Results:z  Overall Accuracy: r  r   r  z  :z    Precision: r  z    Recall: r  z    F1: r  z    Support: r  z
Top 20 Features:r   r  z: r  z.2f	cv_scorescv_meancv_stdtrain_samplestest_samplesr   z=
============================================================zTRAINING COMPLETE)r)   rB   r   r   notnaallr:   r   r1   r  r  itemsre   r!  r   headiterrowsfloatr   rK   rb   r8  )r?   r   r   	valid_idx	split_idxr   r   r  r  r  r;  r4  r-   r   r  irowr6  s                     rA   mainrJ  y  s   	(OOO	
	(OOO	GGG 
B r""H 2bEBBBF   $$!$,,v||~~=I	"HIF	IB	
/c(mm
/
/
/000 CMMC'((ImJYJ'Gk*9*%G]9::&F[$F	
,CLL
,
,
,
,---	
*3v;;
*
*
*
*+++ #7G44E9 

'(((UFF33G	
	
:!4
:
:
:;;;,2244 5 5f"4::<<"""###9{ 3999:::3VH-333444++++,,,3fY/3334444 

'x/?@@J//"%%..00 > >3<3y><<S%6<<<==== %GKrwy1122GIbfY//00GH"7||GO!&kkGNh.//GJE8W55J	/	
	(OOO("G++rC   __main__)r   )r   r   r   )r   r   )r   r   )N)r"  )1__doc__pandasr.   numpyr   r   r   pathlibr   r/  warningsfilterwarningslightgbmr   sklearn.model_selectionr   sklearn.metricsr   r	   r
   r   r-  r+   r+  r,  r   r   r   r   r   r   r   rB   rR   r`   rg   rq   r   r   r   r  r  r!  r8  rJ  __name__r  r  r4   rC   rA   <module>rW     sH            ( ( ( ( ( ( ( (           ! ! !     3 3 3 3 3 3 S S S S S S S S S S S S  4677D9::	 	    -,, 
				
  D" " " "( ( ( (   , , , ,P P Pf   &   >; ; ; ;|! ! !H     6F, F, F,R z$(DFF!E=''' rC   