stable/freqtrade/freqai/prediction_models/RL/RLPrediction_env.py
2022-08-24 13:00:55 +02:00

648 lines
23 KiB
Python

import logging
import random
from collections import deque
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
import gym
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
from gym import spaces
from gym.utils import seeding
from sklearn.decomposition import PCA, KernelPCA
logger = logging.getLogger(__name__)
# from bokeh.io import output_notebook
# from bokeh.plotting import figure, show
# from bokeh.models import (
# CustomJS,
# ColumnDataSource,
# NumeralTickFormatter,
# Span,
# HoverTool,
# Range1d,
# DatetimeTickFormatter,
# Scatter,
# Label, LabelSet
# )
class Actions(Enum):
Short = 0
Long = 1
Neutral = 2
class Actions_v2(Enum):
Neutral = 0
Long_buy = 1
Long_sell = 2
Short_buy = 3
Short_sell = 4
class Positions(Enum):
Short = 0
Long = 1
Neutral = 0.5
def opposite(self):
return Positions.Short if self == Positions.Long else Positions.Long
def mean_over_std(x):
std = np.std(x, ddof=1)
mean = np.mean(x)
return mean / std if std > 0 else 0
class DEnv(gym.Env):
metadata = {'render.modes': ['human']}
def __init__(self, df, prices, reward_kwargs, window_size=10, starting_point=True, ):
assert df.ndim == 2
self.seed()
self.df = df
self.signal_features = self.df
self.prices = prices
self.window_size = window_size
self.starting_point = starting_point
self.rr = reward_kwargs["rr"]
self.profit_aim = reward_kwargs["profit_aim"]
self.fee=0.0015
# # spaces
self.shape = (window_size, self.signal_features.shape[1])
self.action_space = spaces.Discrete(len(Actions_v2))
self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=self.shape, dtype=np.float32)
# episode
self._start_tick = self.window_size
self._end_tick = len(self.prices) - 1
self._done = None
self._current_tick = None
self._last_trade_tick = None
self._position = Positions.Neutral
self._position_history = None
self.total_reward = None
self._total_profit = None
self._first_rendering = None
self.history = None
self.trade_history = []
# self.A_t, self.B_t = 0.000639, 0.00001954
self.r_t_change = 0.
self.returns_report = []
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def reset(self):
self._done = False
if self.starting_point == True:
self._position_history = (self._start_tick* [None]) + [self._position]
else:
self._position_history = (self.window_size * [None]) + [self._position]
self._current_tick = self._start_tick
self._last_trade_tick = None
#self._last_trade_tick = self._current_tick - 1
self._position = Positions.Neutral
self.total_reward = 0.
self._total_profit = 1. # unit
self._first_rendering = True
self.history = {}
self.trade_history = []
self.portfolio_log_returns = np.zeros(len(self.prices))
self._profits = [(self._start_tick, 1)]
self.close_trade_profit = []
self.r_t_change = 0.
self.returns_report = []
return self._get_observation()
def step(self, action):
self._done = False
self._current_tick += 1
if self._current_tick == self._end_tick:
self._done = True
self.update_portfolio_log_returns(action)
self._update_profit(action)
step_reward = self._calculate_reward(action)
self.total_reward += step_reward
trade_type = None
if self.is_tradesignal_v2(action): # exclude 3 case not trade
# Update position
"""
Action: Neutral, position: Long -> Close Long
Action: Neutral, position: Short -> Close Short
Action: Long, position: Neutral -> Open Long
Action: Long, position: Short -> Close Short and Open Long
Action: Short, position: Neutral -> Open Short
Action: Short, position: Long -> Close Long and Open Short
"""
temp_position = self._position
if action == Actions_v2.Neutral.value:
self._position = Positions.Neutral
trade_type = "neutral"
elif action == Actions_v2.Long_buy.value:
self._position = Positions.Long
trade_type = "long"
elif action == Actions_v2.Short_buy.value:
self._position = Positions.Short
trade_type = "short"
elif action == Actions_v2.Long_sell.value:
self._position = Positions.Neutral
trade_type = "neutral"
elif action == Actions_v2.Short_sell.value:
self._position = Positions.Neutral
trade_type = "neutral"
else:
print("case not defined")
# Update last trade tick
self._last_trade_tick = self._current_tick
if trade_type != None:
self.trade_history.append(
{'price': self.current_price(), 'index': self._current_tick, 'type': trade_type})
if self._total_profit < 0.2:
self._done = True
self._position_history.append(self._position)
observation = self._get_observation()
info = dict(
tick = self._current_tick,
total_reward = self.total_reward,
total_profit = self._total_profit,
position = self._position.value
)
self._update_history(info)
return observation, step_reward, self._done, info
def processState(self, state):
return state.to_numpy()
def convert_mlp_Policy(self, obs_):
pass
def _get_observation(self):
return self.signal_features[(self._current_tick - self.window_size):self._current_tick]
def get_unrealized_profit(self):
if self._last_trade_tick == None:
return 0.
if self._position == Positions.Neutral:
return 0.
elif self._position == Positions.Short:
current_price = self.add_buy_fee(self.prices.iloc[self._current_tick].open)
last_trade_price = self.add_sell_fee(self.prices.iloc[self._last_trade_tick].open)
return (last_trade_price - current_price)/last_trade_price
elif self._position == Positions.Long:
current_price = self.add_sell_fee(self.prices.iloc[self._current_tick].open)
last_trade_price = self.add_buy_fee(self.prices.iloc[self._last_trade_tick].open)
return (current_price - last_trade_price)/last_trade_price
else:
return 0.
def is_tradesignal(self, action):
# trade signal
"""
not trade signal is :
Action: Neutral, position: Neutral -> Nothing
Action: Long, position: Long -> Hold Long
Action: Short, position: Short -> Hold Short
"""
return not ((action == Actions.Neutral.value and self._position == Positions.Neutral)
or (action == Actions.Short.value and self._position == Positions.Short)
or (action == Actions.Long.value and self._position == Positions.Long))
def is_tradesignal_v2(self, action):
# trade signal
"""
not trade signal is :
Action: Neutral, position: Neutral -> Nothing
Action: Long, position: Long -> Hold Long
Action: Short, position: Short -> Hold Short
"""
return not ((action == Actions_v2.Neutral.value and self._position == Positions.Neutral) or
(action == Actions_v2.Short_buy.value and self._position == Positions.Short) or
(action == Actions_v2.Short_sell.value and self._position == Positions.Short) or
(action == Actions_v2.Short_buy.value and self._position == Positions.Long) or
(action == Actions_v2.Short_sell.value and self._position == Positions.Long) or
(action == Actions_v2.Long_buy.value and self._position == Positions.Long) or
(action == Actions_v2.Long_sell.value and self._position == Positions.Long) or
(action == Actions_v2.Long_buy.value and self._position == Positions.Short) or
(action == Actions_v2.Long_sell.value and self._position == Positions.Short))
def _is_trade(self, action: Actions):
return ((action == Actions.Long.value and self._position == Positions.Short) or
(action == Actions.Short.value and self._position == Positions.Long) or
(action == Actions.Neutral.value and self._position == Positions.Long) or
(action == Actions.Neutral.value and self._position == Positions.Short)
)
def _is_trade_v2(self, action: Actions_v2):
return ((action == Actions_v2.Long_buy.value and self._position == Positions.Short) or
(action == Actions_v2.Short_buy.value and self._position == Positions.Long) or
(action == Actions_v2.Neutral.value and self._position == Positions.Long) or
(action == Actions_v2.Neutral.value and self._position == Positions.Short) or
(action == Actions_v2.Neutral.Short_sell and self._position == Positions.Long) or
(action == Actions_v2.Neutral.Long_sell and self._position == Positions.Short)
)
def is_hold(self, action):
return ((action == Actions.Short.value and self._position == Positions.Short)
or (action == Actions.Long.value and self._position == Positions.Long))
def is_hold_v2(self, action):
return ((action == Actions_v2.Short_buy.value and self._position == Positions.Short)
or (action == Actions_v2.Long_buy.value and self._position == Positions.Long))
def add_buy_fee(self, price):
return price * (1 + self.fee)
def add_sell_fee(self, price):
return price / (1 + self.fee)
def _update_history(self, info):
if not self.history:
self.history = {key: [] for key in info.keys()}
for key, value in info.items():
self.history[key].append(value)
def render(self, mode='human'):
def _plot_position(position, tick):
color = None
if position == Positions.Short:
color = 'red'
elif position == Positions.Long:
color = 'green'
if color:
plt.scatter(tick, self.prices.loc[tick].open, color=color)
if self._first_rendering:
self._first_rendering = False
plt.cla()
plt.plot(self.prices)
start_position = self._position_history[self._start_tick]
_plot_position(start_position, self._start_tick)
plt.cla()
plt.plot(self.prices)
_plot_position(self._position, self._current_tick)
plt.suptitle("Total Reward: %.6f" % self.total_reward + ' ~ ' + "Total Profit: %.6f" % self._total_profit)
plt.pause(0.01)
def render_all(self):
plt.figure()
window_ticks = np.arange(len(self._position_history))
plt.plot(self.prices['open'], alpha=0.5)
short_ticks = []
long_ticks = []
neutral_ticks = []
for i, tick in enumerate(window_ticks):
if self._position_history[i] == Positions.Short:
short_ticks.append(tick - 1)
elif self._position_history[i] == Positions.Long:
long_ticks.append(tick - 1)
elif self._position_history[i] == Positions.Neutral:
neutral_ticks.append(tick - 1)
plt.plot(neutral_ticks, self.prices.loc[neutral_ticks].open,
'o', color='grey', ms=3, alpha=0.1)
plt.plot(short_ticks, self.prices.loc[short_ticks].open,
'o', color='r', ms=3, alpha=0.8)
plt.plot(long_ticks, self.prices.loc[long_ticks].open,
'o', color='g', ms=3, alpha=0.8)
plt.suptitle("Generalising")
fig = plt.gcf()
fig.set_size_inches(15, 10)
def close_trade_report(self):
small_trade = 0
positive_big_trade = 0
negative_big_trade = 0
small_profit = 0.003
for i in self.close_trade_profit:
if i < small_profit and i > -small_profit:
small_trade+=1
elif i > small_profit:
positive_big_trade += 1
elif i < -small_profit:
negative_big_trade += 1
print(f"small trade={small_trade/len(self.close_trade_profit)}; positive_big_trade={positive_big_trade/len(self.close_trade_profit)}; negative_big_trade={negative_big_trade/len(self.close_trade_profit)}")
def report(self):
# get total trade
long_trade = 0
short_trade = 0
neutral_trade = 0
for trade in self.trade_history:
if trade['type'] == 'long':
long_trade += 1
elif trade['type'] == 'short':
short_trade += 1
else:
neutral_trade += 1
negative_trade = 0
positive_trade = 0
for tr in self.close_trade_profit:
if tr < 0.:
negative_trade += 1
if tr > 0.:
positive_trade += 1
total_trade_lr = negative_trade+positive_trade
total_trade = long_trade + short_trade
sharp_ratio = self.sharpe_ratio()
sharp_log = self.get_sharpe_ratio()
from tabulate import tabulate
headers = ["Performance", ""]
performanceTable = [["Total Trade", "{0:.2f}".format(total_trade)],
["Total reward", "{0:.3f}".format(self.total_reward)],
["Start profit(unit)", "{0:.2f}".format(1.)],
["End profit(unit)", "{0:.3f}".format(self._total_profit)],
["Sharp ratio", "{0:.3f}".format(sharp_ratio)],
["Sharp log", "{0:.3f}".format(sharp_log)],
# ["Sortino ratio", "{0:.2f}".format(0) + '%'],
["winrate", "{0:.2f}".format(positive_trade*100/total_trade_lr) + '%']
]
tabulation = tabulate(performanceTable, headers, tablefmt="fancy_grid", stralign="center")
print(tabulation)
result = {
"Start": "{0:.2f}".format(1.),
"End": "{0:.2f}".format(self._total_profit),
"Sharp": "{0:.3f}".format(sharp_ratio),
"Winrate": "{0:.2f}".format(positive_trade*100/total_trade_lr)
}
return result
def close(self):
plt.close()
def get_sharpe_ratio(self):
return mean_over_std(self.get_portfolio_log_returns())
def save_rendering(self, filepath):
plt.savefig(filepath)
def pause_rendering(self):
plt.show()
def _calculate_reward(self, action):
# rw = self.transaction_profit_reward(action)
#rw = self.reward_rr_profit_config(action)
rw = self.reward_rr_profit_config_v2(action)
return rw
def _update_profit(self, action):
#if self._is_trade(action) or self._done:
if self._is_trade_v2(action) or self._done:
pnl = self.get_unrealized_profit()
if self._position == Positions.Long:
self._total_profit = self._total_profit + self._total_profit*pnl
self._profits.append((self._current_tick, self._total_profit))
self.close_trade_profit.append(pnl)
if self._position == Positions.Short:
self._total_profit = self._total_profit + self._total_profit*pnl
self._profits.append((self._current_tick, self._total_profit))
self.close_trade_profit.append(pnl)
def most_recent_return(self, action):
"""
We support Long, Neutral and Short positions.
Return is generated from rising prices in Long
and falling prices in Short positions.
The actions Sell/Buy or Hold during a Long position trigger the sell/buy-fee.
"""
# Long positions
if self._position == Positions.Long:
current_price = self.prices.iloc[self._current_tick].open
#if action == Actions.Short.value or action == Actions.Neutral.value:
if action == Actions_v2.Short_buy.value or action == Actions_v2.Neutral.value:
current_price = self.add_sell_fee(current_price)
previous_price = self.prices.iloc[self._current_tick - 1].open
if (self._position_history[self._current_tick - 1] == Positions.Short
or self._position_history[self._current_tick - 1] == Positions.Neutral):
previous_price = self.add_buy_fee(previous_price)
return np.log(current_price) - np.log(previous_price)
# Short positions
if self._position == Positions.Short:
current_price = self.prices.iloc[self._current_tick].open
#if action == Actions.Long.value or action == Actions.Neutral.value:
if action == Actions_v2.Long_buy.value or action == Actions_v2.Neutral.value:
current_price = self.add_buy_fee(current_price)
previous_price = self.prices.iloc[self._current_tick - 1].open
if (self._position_history[self._current_tick - 1] == Positions.Long
or self._position_history[self._current_tick - 1] == Positions.Neutral):
previous_price = self.add_sell_fee(previous_price)
return np.log(previous_price) - np.log(current_price)
return 0
def get_portfolio_log_returns(self):
return self.portfolio_log_returns[1:self._current_tick + 1]
def get_trading_log_return(self):
return self.portfolio_log_returns[self._start_tick:]
def update_portfolio_log_returns(self, action):
self.portfolio_log_returns[self._current_tick] = self.most_recent_return(action)
def current_price(self) -> float:
return self.prices.iloc[self._current_tick].open
def prev_price(self) -> float:
return self.prices.iloc[self._current_tick-1].open
def sharpe_ratio(self):
if len(self.close_trade_profit) == 0:
return 0.
returns = np.array(self.close_trade_profit)
reward = (np.mean(returns) - 0. + 1e-9) / (np.std(returns) + 1e-9)
return reward
def get_bnh_log_return(self):
return np.diff(np.log(self.prices['open'][self._start_tick:]))
def transaction_profit_reward(self, action):
rw = 0.
pt = self.prev_price()
pt_1 = self.current_price()
if self._position == Positions.Long:
a_t = 1
elif self._position == Positions.Short:
a_t = -1
else:
a_t = 0
# close long
if (action == Actions.Short.value or action == Actions.Neutral.value) and self._position == Positions.Long:
pt_1 = self.add_sell_fee(self.current_price())
po = self.add_buy_fee(self.prices.iloc[self._last_trade_tick].open)
rw = a_t*(pt_1 - po)/po
#rw = rw*2
# close short
elif (action == Actions.Long.value or action == Actions.Neutral.value) and self._position == Positions.Short:
pt_1 = self.add_buy_fee(self.current_price())
po = self.add_sell_fee(self.prices.iloc[self._last_trade_tick].open)
rw = a_t*(pt_1 - po)/po
#rw = rw*2
else:
rw = a_t*(pt_1 - pt)/pt
return np.clip(rw, 0, 1)
def reward_rr_profit_config_v2(self, action):
rw = 0.
pt_1 = self.current_price()
if len(self.close_trade_profit) > 0:
# long
if self._position == Positions.Long:
pt_1 = self.add_sell_fee(self.current_price())
po = self.add_buy_fee(self.prices.iloc[self._last_trade_tick].open)
if action == Actions_v2.Short_buy.value:
if self.close_trade_profit[-1] > self.profit_aim * self.rr:
rw = 10 * 2
elif self.close_trade_profit[-1] > 0 and self.close_trade_profit[-1] < self.profit_aim * self.rr:
rw = 10 * 1 * 1
elif self.close_trade_profit[-1] < 0:
rw = 10 * -1
elif self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 3 * -1
if action == Actions_v2.Long_sell.value:
if self.close_trade_profit[-1] > self.profit_aim * self.rr:
rw = 10 * 5
elif self.close_trade_profit[-1] > 0 and self.close_trade_profit[-1] < self.profit_aim * self.rr:
rw = 10 * 1 * 3
elif self.close_trade_profit[-1] < 0:
rw = 10 * -1
elif self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 3 * -1
if action == Actions_v2.Neutral.value:
if self.close_trade_profit[-1] > 0:
rw = 2
elif self.close_trade_profit[-1] < 0:
rw = 2 * -1
# short
if self._position == Positions.Short:
pt_1 = self.add_sell_fee(self.current_price())
po = self.add_buy_fee(self.prices.iloc[self._last_trade_tick].open)
if action == Actions_v2.Long_buy.value:
if self.close_trade_profit[-1] > self.profit_aim * self.rr:
rw = 10 * 2
elif self.close_trade_profit[-1] > 0 and self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 1 * 1
elif self.close_trade_profit[-1] < 0:
rw = 10 * -1
elif self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 3 * -1
if action == Actions_v2.Short_sell.value:
if self.close_trade_profit[-1] > self.profit_aim * self.rr:
rw = 10 * 5
elif self.close_trade_profit[-1] > 0 and self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 1 * 3
elif self.close_trade_profit[-1] < 0:
rw = 10 * -1
elif self.close_trade_profit[-1] < (self.profit_aim * -1) * self.rr:
rw = 10 * 3 * -1
if action == Actions_v2.Neutral.value:
if self.close_trade_profit[-1] > 0:
rw = 2
elif self.close_trade_profit[-1] < 0:
rw = 2 * -1
return np.clip(rw, 0, 1)