adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into db_keep_orders

Browse Source
This commit is contained in:
Matthias
2020-08-23 10:36:56 +02:00
parent 674b510d23 92d8adf36c
commit 4ecb67d1d1
61 changed files with 1415 additions and 740 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/commands/arguments.py
View File

@@ -366,7 +366,7 @@ class Arguments:
plot_profit_cmd = subparsers.add_parser(
'plot-profit',
help='Generate plot showing profits.',
parents=[_common_parser],
parents=[_common_parser, _strategy_parser],
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)

4
freqtrade/commands/list_commands.py
View File

@@ -14,7 +14,7 @@ from freqtrade.configuration import setup_utils_configuration
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
market_is_active, symbol_is_pair)
market_is_active)
from freqtrade.misc import plural
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
@@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
'Base': v['base'], 'Quote': v['quote'],
'Active': market_is_active(v),
**({'Is pair': symbol_is_pair(v['symbol'])}
**({'Is pair': exchange.market_is_tradable(v)}
if not pairs_only else {})})
if (args.get('print_one_column', False) or

2
freqtrade/configuration/configuration.py
View File

@@ -199,7 +199,7 @@ class Configuration:
config['exportfilename'] = Path(config['exportfilename'])
else:
config['exportfilename'] = (config['user_data_dir']
/ 'backtest_results/backtest-result.json')
/ 'backtest_results')
def _process_optimize_options(self, config: Dict[str, Any]) -> None:

3
freqtrade/constants.py
View File

@@ -26,12 +26,15 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'ShuffleFilter', 'SpreadFilter']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
DRY_RUN_WALLET = 1000
DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons
DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
# Don't modify sequence of DEFAULT_TRADES_COLUMNS
# it has wide consequences for stored trades files
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
LAST_BT_RESULT_FN = '.last_result.json'
USERPATH_HYPEROPTS = 'hyperopts'
USERPATH_STRATEGIES = 'strategies'
USERPATH_NOTEBOOKS = 'notebooks'

172
freqtrade/data/btanalysis.py
View File

@@ -3,52 +3,123 @@ Helpers when analyzing backtest data
"""
import logging
from pathlib import Path
from typing import Dict, Union, Tuple
from typing import Dict, Union, Tuple, Any, Optional
import numpy as np
import pandas as pd
from datetime import timezone
from freqtrade import persistence
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.misc import json_load
from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
# must align with columns in backtest.py
BT_DATA_COLUMNS = ["pair", "profit_percent", "open_time", "close_time", "index", "duration",
BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
"open_rate", "close_rate", "open_at_end", "sell_reason"]
def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame:
def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
"""
Load backtest data file.
:param filename: pathlib.Path object, or string pointing to the file.
:return: a dataframe with the analysis results
Get latest backtest export based on '.last_result.json'.
:param directory: Directory to search for last result
:return: string containing the filename of the latest backtest result
:raises: ValueError in the following cases:
* Directory does not exist
* `directory/.last_result.json` does not exist
* `directory/.last_result.json` has the wrong content
"""
if isinstance(filename, str):
filename = Path(filename)
if isinstance(directory, str):
directory = Path(directory)
if not directory.is_dir():
raise ValueError(f"Directory '{directory}' does not exist.")
filename = directory / LAST_BT_RESULT_FN
if not filename.is_file():
raise ValueError(f"File {filename} does not exist.")
raise ValueError(
f"Directory '{directory}' does not seem to contain backtest statistics yet.")
with filename.open() as file:
data = json_load(file)
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
if 'latest_backtest' not in data:
raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.")
df['open_time'] = pd.to_datetime(df['open_time'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_time'] = pd.to_datetime(df['close_time'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['profit'] = df['close_rate'] - df['open_rate']
df = df.sort_values("open_time").reset_index(drop=True)
return data['latest_backtest']
def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
"""
Load backtest statistics file.
:param filename: pathlib.Path object, or string pointing to the file.
:return: a dictionary containing the resulting file.
"""
if isinstance(filename, str):
filename = Path(filename)
if filename.is_dir():
filename = filename / get_latest_backtest_filename(filename)
if not filename.is_file():
raise ValueError(f"File {filename} does not exist.")
logger.info(f"Loading backtest result from {filename}")
with filename.open() as file:
data = json_load(file)
return data
def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load backtest data file.
:param filename: pathlib.Path object, or string pointing to a file or directory
:param strategy: Strategy to load - mainly relevant for multi-strategy backtests
Can also serve as protection to load the correct result.
:return: a dataframe with the analysis results
:raise: ValueError if loading goes wrong.
"""
data = load_backtest_stats(filename)
if not isinstance(data, list):
# new, nested format
if 'strategy' not in data:
raise ValueError("Unknown dataformat.")
if not strategy:
if len(data['strategy']) == 1:
strategy = list(data['strategy'].keys())[0]
else:
raise ValueError("Detected backtest result with more than one strategy. "
"Please specify a strategy.")
if strategy not in data['strategy']:
raise ValueError(f"Strategy {strategy} not available in the backtest result.")
data = data['strategy'][strategy]['trades']
df = pd.DataFrame(data)
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
else:
# old format - only with lists.
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['profit_abs'] = df['close_rate'] - df['open_rate']
df = df.sort_values("open_date").reset_index(drop=True)
return df
@@ -62,9 +133,9 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
"""
from freqtrade.exchange import timeframe_to_minutes
timeframe_min = timeframe_to_minutes(timeframe)
dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time,
dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
freq=f"{timeframe_min}min"))
for row in results[['open_time', 'close_time']].iterrows()]
for row in results[['open_date', 'close_date']].iterrows()]
deltas = [len(x) for x in dates]
dates = pd.Series(pd.concat(dates).values, name='date')
df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
@@ -90,21 +161,26 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
return df_final[df_final['open_trades'] > max_open_trades]
def load_trades_from_db(db_url: str) -> pd.DataFrame:
def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load trades from a DB (using dburl)
:param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite)
:param strategy: Strategy to load - mainly relevant for multi-strategy backtests
Can also serve as protection to load the correct result.
:return: Dataframe containing Trades
"""
trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS)
persistence.init(db_url, clean_open_orders=False)
columns = ["pair", "open_time", "close_time", "profit", "profit_percent",
"open_rate", "close_rate", "amount", "duration", "sell_reason",
columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
"open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
"fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
"stake_amount", "max_rate", "min_rate", "id", "exchange",
"stop_loss", "initial_stop_loss", "strategy", "timeframe"]
filters = []
if strategy:
filters.append(Trade.strategy == strategy)
trades = pd.DataFrame([(t.pair,
t.open_date.replace(tzinfo=timezone.utc),
t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
@@ -123,14 +199,14 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
t.stop_loss, t.initial_stop_loss,
t.strategy, t.timeframe
)
for t in Trade.get_trades().all()],
for t in Trade.get_trades(filters).all()],
columns=columns)
return trades
def load_trades(source: str, db_url: str, exportfilename: Path,
no_trades: bool = False) -> pd.DataFrame:
no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame:
"""
Based on configuration option "trade_source":
* loads data from DB (using `db_url`)
@@ -148,7 +224,7 @@ def load_trades(source: str, db_url: str, exportfilename: Path,
if source == "DB":
return load_trades_from_db(db_url)
elif source == "file":
return load_backtest_data(exportfilename)
return load_backtest_data(exportfilename, strategy)
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
@@ -163,11 +239,31 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
else:
trades_start = dataframe.iloc[0]['date']
trades_stop = dataframe.iloc[-1]['date']
trades = trades.loc[(trades['open_time'] >= trades_start) &
(trades['close_time'] <= trades_stop)]
trades = trades.loc[(trades['open_date'] >= trades_start) &
(trades['close_date'] <= trades_stop)]
return trades
def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
"""
Calculate market change based on "column".
Calculation is done by taking the first non-null and the last non-null element of each column
and calculating the pctchange as "(last - first) / first".
Then the results per pair are combined as mean.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return:
"""
tmp_means = []
for pair, df in data.items():
start = df[column].dropna().iloc[0]
end = df[column].dropna().iloc[-1]
tmp_means.append((end - start) / start)
return np.mean(tmp_means)
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
@@ -190,7 +286,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_time and profit_percent)
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
@@ -201,7 +297,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_time'
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_percent']].sum()
df.loc[:, col_name] = _trades_sum.cumsum()
# Set first value to 0
@@ -211,13 +307,13 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
return df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time',
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_percent'
) -> Tuple[float, pd.Timestamp, pd.Timestamp]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_time and profit_percent)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_time')
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_percent')
:return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time
:raise: ValueError if trade-dataframe was found empty.

19
freqtrade/edge/edge_positioning.py
View File

@@ -9,7 +9,7 @@ import utils_find_1st as utf1st
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, DATETIME_PRINT_FORMAT
from freqtrade.exceptions import OperationalException
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.strategy.interface import SellType
@@ -121,12 +121,9 @@ class Edge:
# Print timeframe
min_date, max_date = get_timerange(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days) ...',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
)
logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
trades: list = []
@@ -240,7 +237,7 @@ class Edge:
# All returned values are relative, they are defined as ratios.
stake = 0.015
result['trade_duration'] = result['close_time'] - result['open_time']
result['trade_duration'] = result['close_date'] - result['open_date']
result['trade_duration'] = result['trade_duration'].map(
lambda x: int(x.total_seconds() / 60))
@@ -430,10 +427,8 @@ class Edge:
'stoploss': stoploss,
'profit_ratio': '',
'profit_abs': '',
'open_time': date_column[open_trade_index],
'close_time': date_column[exit_index],
'open_index': start_point + open_trade_index,
'close_index': start_point + exit_index,
'open_date': date_column[open_trade_index],
'close_date': date_column[exit_index],
'trade_duration': '',
'open_rate': round(open_price, 15),
'close_rate': round(exit_price, 15),

3
freqtrade/exchange/__init__.py
View File

@@ -12,8 +12,7 @@ from freqtrade.exchange.exchange import (timeframe_to_seconds,
timeframe_to_msecs,
timeframe_to_next_date,
timeframe_to_prev_date)
from freqtrade.exchange.exchange import (market_is_active,
symbol_is_pair)
from freqtrade.exchange.exchange import (market_is_active)
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bibox import Bibox

29
freqtrade/exchange/exchange.py
View File

@@ -223,7 +223,7 @@ class Exchange:
if quote_currencies:
markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies}
if pairs_only:
markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])}
markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)}
if active_only:
markets = {k: v for k, v in markets.items() if market_is_active(v)}
return markets
@@ -247,6 +247,19 @@ class Exchange:
"""
return self.markets.get(pair, {}).get('base', '')
def market_is_tradable(self, market: Dict[str, Any]) -> bool:
"""
Check if the market symbol is tradable by Freqtrade.
By default, checks if it's splittable by `/` and both sides correspond to base / quote
"""
symbol_parts = market['symbol'].split('/')
return (len(symbol_parts) == 2 and
len(symbol_parts[0]) > 0 and
len(symbol_parts[1]) > 0 and
symbol_parts[0] == market.get('base') and
symbol_parts[1] == market.get('quote')
)
def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame:
if pair_interval in self._klines:
return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -1271,20 +1284,6 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
def symbol_is_pair(market_symbol: str, base_currency: str = None,
quote_currency: str = None) -> bool:
"""
Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the
quote currency separated by '/' character. If base_currency and/or quote_currency is passed,
it also checks that the symbol contains appropriate base and/or quote currency part before
and after the separating character correspondingly.
"""
symbol_parts = market_symbol.split('/')
return (len(symbol_parts) == 2 and
(symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and
(symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0))
def market_is_active(market: Dict) -> bool:
"""
Return True if the market is active.

12
freqtrade/exchange/ftx.py
View File

@@ -1,6 +1,6 @@
""" FTX exchange subclass """
import logging
from typing import Dict
from typing import Any, Dict
import ccxt
@@ -20,6 +20,16 @@ class Ftx(Exchange):
"ohlcv_candle_limit": 1500,
}
def market_is_tradable(self, market: Dict[str, Any]) -> bool:
"""
Check if the market symbol is tradable by Freqtrade.
Default checks + check if pair is spot pair (no futures trading yet).
"""
parent_check = super().market_is_tradable(market)
return (parent_check and
market.get('spot', False) is True)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)

12
freqtrade/exchange/kraken.py
View File

@@ -1,6 +1,6 @@
""" Kraken exchange subclass """
import logging
from typing import Dict
from typing import Any, Dict
import ccxt
@@ -22,6 +22,16 @@ class Kraken(Exchange):
"trades_pagination_arg": "since",
}
def market_is_tradable(self, market: Dict[str, Any]) -> bool:
"""
Check if the market symbol is tradable by Freqtrade.
Default checks + check if pair is darkpool pair.
"""
parent_check = super().market_is_tradable(market)
return (parent_check and
market.get('darkpool', False) is False)
@retrier
def get_balances(self) -> dict:
if self._config['dry_run']:

75
freqtrade/optimize/backtesting.py
View File

@@ -13,6 +13,7 @@ from pandas import DataFrame
from freqtrade.configuration import (TimeRange, remove_credentials,
validate_config_consistency)
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
@@ -20,11 +21,10 @@ from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
show_backtest_results,
store_backtest_result)
store_backtest_stats)
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
logger = logging.getLogger(__name__)
@@ -37,14 +37,15 @@ class BacktestResult(NamedTuple):
pair: str
profit_percent: float
profit_abs: float
open_time: datetime
close_time: datetime
open_index: int
close_index: int
open_date: datetime
open_rate: float
open_fee: float
close_date: datetime
close_rate: float
close_fee: float
amount: float
trade_duration: float
open_at_end: bool
open_rate: float
close_rate: float
sell_reason: SellType
@@ -65,9 +66,8 @@ class Backtesting:
self.strategylist: List[IStrategy] = []
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
if self.config.get('runmode') != RunMode.HYPEROPT:
self.dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = self.dataprovider
dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = dataprovider
if self.config.get('strategy_list', None):
for strat in list(self.config['strategy_list']):
@@ -137,10 +137,10 @@ class Backtesting:
min_date, max_date = history.get_timerange(data)
logger.info(
'Loading data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
@@ -225,7 +225,7 @@ class Backtesting:
open_rate=buy_row.open,
open_date=buy_row.date,
stake_amount=stake_amount,
amount=stake_amount / buy_row.open,
amount=round(stake_amount / buy_row.open, 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
@@ -246,14 +246,15 @@ class Backtesting:
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=closerate),
profit_abs=trade.calc_profit(rate=closerate),
open_time=buy_row.date,
close_time=sell_row.date,
trade_duration=trade_dur,
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=False,
open_date=buy_row.date,
open_rate=buy_row.open,
open_fee=self.fee,
close_date=sell_row.date,
close_rate=closerate,
close_fee=self.fee,
amount=trade.amount,
trade_duration=trade_dur,
open_at_end=False,
sell_reason=sell.sell_type
)
if partial_ohlcv:
@@ -262,15 +263,16 @@ class Backtesting:
bt_res = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
open_time=buy_row.date,
close_time=sell_row.date,
open_date=buy_row.date,
open_rate=buy_row.open,
open_fee=self.fee,
close_date=sell_row.date,
close_rate=sell_row.open,
close_fee=self.fee,
amount=trade.amount,
trade_duration=int((
sell_row.date - buy_row.date).total_seconds() // 60),
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=True,
open_rate=buy_row.open,
close_rate=sell_row.open,
sell_reason=SellType.FORCE_SELL
)
logger.debug(f"{pair} - Force selling still open trade, "
@@ -356,8 +358,8 @@ class Backtesting:
if trade_entry:
logger.debug(f"{pair} - Locking pair till "
f"close_time={trade_entry.close_time}")
lock_pair_until[pair] = trade_entry.close_time
f"close_date={trade_entry.close_date}")
lock_pair_until[pair] = trade_entry.close_date
trades.append(trade_entry)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
@@ -400,10 +402,9 @@ class Backtesting:
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
'Backtesting with data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
# Execute backtest and print results
all_results[self.strategy.get_strategy_name()] = self.backtest(
processed=preprocessed,
@@ -414,8 +415,10 @@ class Backtesting:
position_stacking=position_stacking,
)
stats = generate_backtest_stats(self.config, data, all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', False):
store_backtest_result(self.config['exportfilename'], all_results)
store_backtest_stats(self.config['exportfilename'], stats)
# Show backtest results
stats = generate_backtest_stats(self.config, data, all_results)
show_backtest_results(self.config, stats)

202
freqtrade/optimize/default_hyperopt.py
View File

@@ -1,202 +0,0 @@
# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
from functools import reduce
from typing import Any, Callable, Dict, List
import talib.abstract as ta
from pandas import DataFrame
from skopt.space import Categorical, Dimension, Integer
import freqtrade.vendor.qtpylib.indicators as qtpylib
from freqtrade.optimize.hyperopt_interface import IHyperOpt
class DefaultHyperOpt(IHyperOpt):
"""
Default hyperopt provided by the Freqtrade bot.
You can override it with your own Hyperopt
"""
@staticmethod
def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Add several indicators needed for buy and sell strategies defined below.
"""
# ADX
dataframe['adx'] = ta.ADX(dataframe)
# MACD
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
dataframe['macdsignal'] = macd['macdsignal']
# MFI
dataframe['mfi'] = ta.MFI(dataframe)
# RSI
dataframe['rsi'] = ta.RSI(dataframe)
# Stochastic Fast
stoch_fast = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch_fast['fastd']
# Minus-DI
dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# Bollinger bands
bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
dataframe['bb_lowerband'] = bollinger['lower']
dataframe['bb_upperband'] = bollinger['upper']
# SAR
dataframe['sar'] = ta.SAR(dataframe)
return dataframe
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the buy strategy parameters to be used by Hyperopt.
"""
def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Buy strategy Hyperopt will build and use.
"""
conditions = []
# GUARDS AND TRENDS
if 'mfi-enabled' in params and params['mfi-enabled']:
conditions.append(dataframe['mfi'] < params['mfi-value'])
if 'fastd-enabled' in params and params['fastd-enabled']:
conditions.append(dataframe['fastd'] < params['fastd-value'])
if 'adx-enabled' in params and params['adx-enabled']:
conditions.append(dataframe['adx'] > params['adx-value'])
if 'rsi-enabled' in params and params['rsi-enabled']:
conditions.append(dataframe['rsi'] < params['rsi-value'])
# TRIGGERS
if 'trigger' in params:
if params['trigger'] == 'bb_lower':
conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
if params['trigger'] == 'macd_cross_signal':
conditions.append(qtpylib.crossed_above(
dataframe['macd'], dataframe['macdsignal']
))
if params['trigger'] == 'sar_reversal':
conditions.append(qtpylib.crossed_above(
dataframe['close'], dataframe['sar']
))
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'buy'] = 1
return dataframe
return populate_buy_trend
@staticmethod
def indicator_space() -> List[Dimension]:
"""
Define your Hyperopt space for searching buy strategy parameters.
"""
return [
Integer(10, 25, name='mfi-value'),
Integer(15, 45, name='fastd-value'),
Integer(20, 50, name='adx-value'),
Integer(20, 40, name='rsi-value'),
Categorical([True, False], name='mfi-enabled'),
Categorical([True, False], name='fastd-enabled'),
Categorical([True, False], name='adx-enabled'),
Categorical([True, False], name='rsi-enabled'),
Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger')
]
@staticmethod
def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the sell strategy parameters to be used by Hyperopt.
"""
def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Sell strategy Hyperopt will build and use.
"""
conditions = []
# GUARDS AND TRENDS
if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']:
conditions.append(dataframe['mfi'] > params['sell-mfi-value'])
if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']:
conditions.append(dataframe['fastd'] > params['sell-fastd-value'])
if 'sell-adx-enabled' in params and params['sell-adx-enabled']:
conditions.append(dataframe['adx'] < params['sell-adx-value'])
if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']:
conditions.append(dataframe['rsi'] > params['sell-rsi-value'])
# TRIGGERS
if 'sell-trigger' in params:
if params['sell-trigger'] == 'sell-bb_upper':
conditions.append(dataframe['close'] > dataframe['bb_upperband'])
if params['sell-trigger'] == 'sell-macd_cross_signal':
conditions.append(qtpylib.crossed_above(
dataframe['macdsignal'], dataframe['macd']
))
if params['sell-trigger'] == 'sell-sar_reversal':
conditions.append(qtpylib.crossed_above(
dataframe['sar'], dataframe['close']
))
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'sell'] = 1
return dataframe
return populate_sell_trend
@staticmethod
def sell_indicator_space() -> List[Dimension]:
"""
Define your Hyperopt space for searching sell strategy parameters.
"""
return [
Integer(75, 100, name='sell-mfi-value'),
Integer(50, 100, name='sell-fastd-value'),
Integer(50, 100, name='sell-adx-value'),
Integer(60, 100, name='sell-rsi-value'),
Categorical([True, False], name='sell-mfi-enabled'),
Categorical([True, False], name='sell-fastd-enabled'),
Categorical([True, False], name='sell-adx-enabled'),
Categorical([True, False], name='sell-rsi-enabled'),
Categorical(['sell-bb_upper',
'sell-macd_cross_signal',
'sell-sar_reversal'], name='sell-trigger')
]
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Based on TA indicators. Should be a copy of same method from strategy.
Must align to populate_indicators in this file.
Only used when --spaces does not include buy space.
"""
dataframe.loc[
(
(dataframe['close'] < dataframe['bb_lowerband']) &
(dataframe['mfi'] < 16) &
(dataframe['adx'] > 25) &
(dataframe['rsi'] < 21)
),
'buy'] = 1
return dataframe
def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Based on TA indicators. Should be a copy of same method from strategy.
Must align to populate_indicators in this file.
Only used when --spaces does not include sell space.
"""
dataframe.loc[
(
(qtpylib.crossed_above(
dataframe['macdsignal'], dataframe['macd']
)) &
(dataframe['fastd'] > 54)
),
'sell'] = 1
return dataframe

33
freqtrade/optimize/hyperopt.py
View File

@@ -4,27 +4,28 @@
This module contains the hyperopt logic
"""
import io
import locale
import logging
import random
import warnings
from math import ceil
from collections import OrderedDict
from math import ceil
from operator import itemgetter
from pathlib import Path
from pprint import pformat
from typing import Any, Dict, List, Optional
import progressbar
import rapidjson
import tabulate
from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from pandas import DataFrame, json_normalize, isna
import progressbar
import tabulate
from os import path
import io
from pandas import DataFrame, isna, json_normalize
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.history import get_timerange
from freqtrade.exceptions import OperationalException
@@ -32,9 +33,11 @@ from freqtrade.misc import plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import \
IHyperOptLoss # noqa: F401
from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver,
HyperOptResolver)
from freqtrade.strategy import IStrategy
# Suppress scikit-learn FutureWarnings from skopt
with warnings.catch_warnings():
@@ -395,7 +398,7 @@ class Hyperopt:
return
# Verification for overwrite
if path.isfile(csv_file):
if Path(csv_file).is_file():
logger.error(f"CSV file already exists: {csv_file}")
return
@@ -641,15 +644,17 @@ class Hyperopt:
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = get_timerange(data)
logger.info(
'Hyperopting with data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
dump(preprocessed, self.data_pickle_file)
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore
self.backtesting.strategy.dp = None # type: ignore
IStrategy.dp = None # type: ignore
self.epochs = self.load_previous_results(self.results_file)
@@ -660,6 +665,10 @@ class Hyperopt:
self.dimensions: List[Dimension] = self.hyperopt_space()
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.print_colorized:
colorama_init(autoreset=True)
try:
with Parallel(n_jobs=config_jobs) as parallel:
jobs = parallel._effective_n_jobs()

2
freqtrade/optimize/hyperopt_loss_sharpe_daily.py
View File

@@ -43,7 +43,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss):
normalize=True)
sum_daily = (
results.resample(resample_freq, on='close_time').agg(
results.resample(resample_freq, on='close_date').agg(
{"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
)

2
freqtrade/optimize/hyperopt_loss_sortino_daily.py
View File

@@ -45,7 +45,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss):
normalize=True)
sum_daily = (
results.resample(resample_freq, on='close_time').agg(
results.resample(resample_freq, on='close_date').agg(
{"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0)
)

190
freqtrade/optimize/optimize_reports.py
View File

@@ -1,46 +1,40 @@
import logging
from datetime import timedelta
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List
from arrow import Arrow
from pandas import DataFrame
from numpy import int64
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import calculate_max_drawdown, calculate_market_change
from freqtrade.misc import file_dump_json
logger = logging.getLogger(__name__)
def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None:
"""
Stores backtest results to file (one file per strategy)
:param recordfilename: Destination filename
:param all_results: Dict of Dataframes, one results dataframe per strategy
Stores backtest results
:param recordfilename: Path object, which can either be a filename or a directory.
Filenames will be appended with a timestamp right before the suffix
while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
:param stats: Dataframe containing the backtesting statistics
"""
for strategy, results in all_results.items():
records = backtest_result_to_list(results)
if recordfilename.is_dir():
filename = (recordfilename /
f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json')
else:
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
).with_suffix(recordfilename.suffix)
file_dump_json(filename, stats)
if records:
filename = recordfilename
if len(all_results) > 1:
# Inject strategy to filename
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
logger.info(f'Dumping backtest results to {filename}')
file_dump_json(filename, records)
def backtest_result_to_list(results: DataFrame) -> List[List]:
"""
Converts a list of Backtest-results to list
:param results: Dataframe containing results for one strategy
:return: List of Lists containing the trades
"""
return [[t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value]
for index, t in results.iterrows()]
latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
def _get_line_floatfmt() -> List[str]:
@@ -66,11 +60,12 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column:
return {
'key': first_column,
'trades': len(result),
'profit_mean': result['profit_percent'].mean(),
'profit_mean_pct': result['profit_percent'].mean() * 100.0,
'profit_mean': result['profit_percent'].mean() if len(result) > 0 else 0.0,
'profit_mean_pct': result['profit_percent'].mean() * 100.0 if len(result) > 0 else 0.0,
'profit_sum': result['profit_percent'].sum(),
'profit_sum_pct': result['profit_percent'].sum() * 100.0,
'profit_total_abs': result['profit_abs'].sum(),
'profit_total': result['profit_percent'].sum() / max_open_trades,
'profit_total_pct': result['profit_percent'].sum() * 100.0 / max_open_trades,
'duration_avg': str(timedelta(
minutes=round(result['trade_duration'].mean()))
@@ -141,7 +136,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
'profit_sum': profit_sum,
'profit_sum_pct': round(profit_sum * 100, 2),
'profit_total_abs': result['profit_abs'].sum(),
'profit_pct_total': profit_percent_tot,
'profit_total_pct': profit_percent_tot,
}
)
return tabular_data
@@ -189,18 +184,58 @@ def generate_edge_table(results: dict) -> str:
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
if len(results) == 0:
return {
'backtest_best_day': 0,
'backtest_worst_day': 0,
'winning_days': 0,
'draw_days': 0,
'losing_days': 0,
'winner_holding_avg': timedelta(),
'loser_holding_avg': timedelta(),
}
daily_profit = results.resample('1d', on='close_date')['profit_percent'].sum()
worst = min(daily_profit)
best = max(daily_profit)
winning_days = sum(daily_profit > 0)
draw_days = sum(daily_profit == 0)
losing_days = sum(daily_profit < 0)
winning_trades = results.loc[results['profit_percent'] > 0]
losing_trades = results.loc[results['profit_percent'] < 0]
return {
'backtest_best_day': best,
'backtest_worst_day': worst,
'winning_days': winning_days,
'draw_days': draw_days,
'losing_days': losing_days,
'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
if not winning_trades.empty else timedelta()),
'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
if not losing_trades.empty else timedelta()),
}
def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
all_results: Dict[str, DataFrame]) -> Dict[str, Any]:
all_results: Dict[str, DataFrame],
min_date: Arrow, max_date: Arrow
) -> Dict[str, Any]:
"""
:param config: Configuration object used for backtest
:param btdata: Backtest data
:param all_results: backtest result - dictionary with { Strategy: results}.
:param min_date: Backtest start date
:param max_date: Backtest end date
:return:
Dictionary containing results per strategy and a stratgy summary.
"""
stake_currency = config['stake_currency']
max_open_trades = config['max_open_trades']
result: Dict[str, Any] = {'strategy': {}}
market_change = calculate_market_change(btdata, 'close')
for strategy, results in all_results.items():
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
@@ -212,14 +247,58 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame],
max_open_trades=max_open_trades,
results=results.loc[results['open_at_end']],
skip_nan=True)
daily_stats = generate_daily_stats(results)
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
backtest_days = (max_date - min_date).days
strat_stats = {
'trades': backtest_result_to_list(results),
'trades': results.to_dict(orient='records'),
'results_per_pair': pair_results,
'sell_reason_summary': sell_reason_stats,
'left_open_trades': left_open_results,
}
'total_trades': len(results),
'profit_mean': results['profit_percent'].mean() if len(results) > 0 else 0,
'profit_total': results['profit_percent'].sum(),
'profit_total_abs': results['profit_abs'].sum(),
'backtest_start': min_date.datetime,
'backtest_start_ts': min_date.timestamp * 1000,
'backtest_end': max_date.datetime,
'backtest_end_ts': max_date.timestamp * 1000,
'backtest_days': backtest_days,
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
'market_change': market_change,
'pairlist': list(btdata.keys()),
'stake_amount': config['stake_amount'],
'stake_currency': config['stake_currency'],
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
**daily_stats,
}
result['strategy'][strategy] = strat_stats
try:
max_drawdown, drawdown_start, drawdown_end = calculate_max_drawdown(
results, value_col='profit_percent')
strat_stats.update({
'max_drawdown': max_drawdown,
'drawdown_start': drawdown_start,
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
'drawdown_end': drawdown_end,
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
})
except ValueError:
strat_stats.update({
'max_drawdown': 0.0,
'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_start_ts': 0,
'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_end_ts': 0,
})
strategy_results = generate_strategy_metrics(stake_currency=stake_currency,
max_open_trades=max_open_trades,
all_results=all_results)
@@ -273,7 +352,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
output = [[
t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_pct_total'],
t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'],
] for t in sell_reason_stats]
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
@@ -298,6 +377,35 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
def text_table_add_metrics(strat_results: Dict) -> str:
if len(strat_results['trades']) > 0:
min_trade = min(strat_results['trades'], key=lambda x: x['open_date'])
metrics = [
('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
('Total trades', strat_results['total_trades']),
('First trade', min_trade['open_date'].strftime(DATETIME_PRINT_FORMAT)),
('First trade Pair', min_trade['pair']),
('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
('Trades per day', strat_results['trades_per_day']),
('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"),
('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"),
('Days win/draw/lose', f"{strat_results['winning_days']} / "
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
('', ''), # Empty line to improve readability
('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"),
('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
]
return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
else:
return ''
def show_backtest_results(config: Dict, backtest_stats: Dict):
stake_currency = config['stake_currency']
@@ -312,15 +420,21 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
stake_currency=stake_currency)
if isinstance(table, str):
if isinstance(table, str) and len(table) > 0:
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
if isinstance(table, str):
if isinstance(table, str) and len(table) > 0:
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str):
table = text_table_add_metrics(results)
if isinstance(table, str) and len(table) > 0:
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str) and len(table) > 0:
print('=' * len(table.splitlines()[0]))
print()

5
freqtrade/pairlist/AgeFilter.py
View File

@@ -26,12 +26,11 @@ class AgeFilter(IPairList):
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
if self._min_days_listed < 1:
raise OperationalException("AgeFilter requires min_days_listed must be >= 1")
raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
if self._min_days_listed > exchange.ohlcv_candle_limit:
raise OperationalException("AgeFilter requires min_days_listed must not exceed "
raise OperationalException("AgeFilter requires min_days_listed to not exceed "
"exchange max request size "
f"({exchange.ohlcv_candle_limit})")
self._enabled = self._min_days_listed >= 1
@property
def needstickers(self) -> bool:

5
freqtrade/pairlist/IPairList.py
View File

@@ -162,6 +162,11 @@ class IPairList(ABC):
f"{self._exchange.name}. Removing it from whitelist..")
continue
if not self._exchange.market_is_tradable(markets[pair]):
logger.warning(f"Pair {pair} is not tradable with Freqtrade."
"Removing it from whitelist..")
continue
if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
logger.warning(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..")

13
freqtrade/pairlist/PriceFilter.py
View File

@@ -4,6 +4,7 @@ Price pair list filter
import logging
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
from freqtrade.pairlist.IPairList import IPairList
@@ -18,11 +19,17 @@ class PriceFilter(IPairList):
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0)
if self._low_price_ratio < 0:
raise OperationalException("PriceFilter requires low_price_ratio to be >= 0")
self._min_price = pairlistconfig.get('min_price', 0)
if self._min_price < 0:
raise OperationalException("PriceFilter requires min_price to be >= 0")
self._max_price = pairlistconfig.get('max_price', 0)
self._enabled = ((self._low_price_ratio != 0) or
(self._min_price != 0) or
(self._max_price != 0))
if self._max_price < 0:
raise OperationalException("PriceFilter requires max_price to be >= 0")
self._enabled = ((self._low_price_ratio > 0) or
(self._min_price > 0) or
(self._max_price > 0))
@property
def needstickers(self) -> bool:

9
freqtrade/persistence/models.py
View File

@@ -276,7 +276,7 @@ class Trade(_DECL_BASE):
'open_date_hum': arrow.get(self.open_date).humanize(),
'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
'open_timestamp': int(self.open_date.timestamp() * 1000),
'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
'open_rate': self.open_rate,
'open_rate_requested': self.open_rate_requested,
'open_trade_price': round(self.open_trade_price, 8),
@@ -285,7 +285,8 @@ class Trade(_DECL_BASE):
if self.close_date else None),
'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
if self.close_date else None),
'close_timestamp': int(self.close_date.timestamp() * 1000) if self.close_date else None,
'close_timestamp': int(self.close_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
'close_rate': self.close_rate,
'close_rate_requested': self.close_rate_requested,
'close_profit': self.close_profit,
@@ -300,8 +301,8 @@ class Trade(_DECL_BASE):
'stoploss_order_id': self.stoploss_order_id,
'stoploss_last_update': (self.stoploss_last_update.strftime("%Y-%m-%d %H:%M:%S")
if self.stoploss_last_update else None),
'stoploss_last_update_timestamp': (int(self.stoploss_last_update.timestamp() * 1000)
if self.stoploss_last_update else None),
'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
'initial_stop_loss': self.initial_stop_loss, # Deprecated - should not be used
'initial_stop_loss_abs': self.initial_stop_loss,
'initial_stop_loss_ratio': (self.initial_stop_loss_pct

29
freqtrade/plot/plotting.py
View File

@@ -8,7 +8,8 @@ from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (calculate_max_drawdown,
combine_dataframes_with_mean,
create_cum_profit,
extract_trades_of_period, load_trades)
extract_trades_of_period,
load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import load_data
@@ -53,19 +54,22 @@ def init_plotscript(config):
)
no_trades = False
filename = config.get('exportfilename')
if config.get('no_trades', False):
no_trades = True
elif not config['exportfilename'].is_file() and config['trade_source'] == 'file':
logger.warning("Backtest file is missing skipping trades.")
no_trades = True
elif config['trade_source'] == 'file':
if not filename.is_dir() and not filename.is_file():
logger.warning("Backtest file is missing skipping trades.")
no_trades = True
trades = load_trades(
config['trade_source'],
db_url=config.get('db_url'),
exportfilename=config.get('exportfilename'),
no_trades=no_trades
exportfilename=filename,
no_trades=no_trades,
strategy=config.get("strategy"),
)
trades = trim_dataframe(trades, timerange, 'open_time')
trades = trim_dataframe(trades, timerange, 'open_date')
return {"ohlcv": data,
"trades": trades,
@@ -165,10 +169,11 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
if trades is not None and len(trades) > 0:
# Create description for sell summarizing the trade
trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
f"{row['sell_reason']}, {row['duration']} min",
f"{row['sell_reason']}, "
f"{row['trade_duration']} min",
axis=1)
trade_buys = go.Scatter(
x=trades["open_time"],
x=trades["open_date"],
y=trades["open_rate"],
mode='markers',
name='Trade buy',
@@ -183,7 +188,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
)
trade_sells = go.Scatter(
x=trades.loc[trades['profit_percent'] > 0, "close_time"],
x=trades.loc[trades['profit_percent'] > 0, "close_date"],
y=trades.loc[trades['profit_percent'] > 0, "close_rate"],
text=trades.loc[trades['profit_percent'] > 0, "desc"],
mode='markers',
@@ -196,7 +201,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
)
)
trade_sells_loss = go.Scatter(
x=trades.loc[trades['profit_percent'] <= 0, "close_time"],
x=trades.loc[trades['profit_percent'] <= 0, "close_date"],
y=trades.loc[trades['profit_percent'] <= 0, "close_rate"],
text=trades.loc[trades['profit_percent'] <= 0, "desc"],
mode='markers',
@@ -510,7 +515,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
# Remove open pairs - we don't know the profit yet so can't calculate profit for these.
# Also, If only one open pair is left, then the profit-generation would fail.
trades = trades[(trades['pair'].isin(plot_elements["pairs"]))
& (~trades['close_time'].isnull())
& (~trades['close_date'].isnull())
]
if len(trades) == 0:
raise OperationalException("No trades found, cannot generate Profit-plot without "

2
freqtrade/resolvers/hyperopt_resolver.py
View File

@@ -23,7 +23,7 @@ class HyperOptResolver(IResolver):
object_type = IHyperOpt
object_type_str = "Hyperopt"
user_subdir = USERPATH_HYPEROPTS
initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
initial_search_path = None
@staticmethod
def load_hyperopt(config: Dict) -> IHyperOpt:

3
freqtrade/rpc/api_server.py
View File

@@ -16,6 +16,7 @@ from werkzeug.security import safe_str_cmp
from werkzeug.serving import make_server
from freqtrade.__init__ import __version__
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.rpc.rpc import RPC, RPCException
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -32,7 +33,7 @@ class ArrowJSONEncoder(JSONEncoder):
elif isinstance(obj, date):
return obj.strftime("%Y-%m-%d")
elif isinstance(obj, datetime):
return obj.strftime("%Y-%m-%d %H:%M:%S")
return obj.strftime(DATETIME_PRINT_FORMAT)
iterable = iter(obj)
except TypeError:
pass

10
freqtrade/rpc/rpc.py
View File

@@ -224,22 +224,20 @@ class RPC:
]).order_by(Trade.close_date).all()
curdayprofit = sum(trade.close_profit_abs for trade in trades)
profit_days[profitday] = {
'amount': f'{curdayprofit:.8f}',
'amount': curdayprofit,
'trades': len(trades)
}
data = [
{
'date': key,
'abs_profit': f'{float(value["amount"]):.8f}',
'fiat_value': '{value:.3f}'.format(
value=self._fiat_converter.convert_amount(
'abs_profit': value["amount"],
'fiat_value': self._fiat_converter.convert_amount(
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
),
'trade_count': f'{value["trades"]}',
'trade_count': value["trades"],
}
for key, value in profit_days.items()
]

4
freqtrade/rpc/telegram.py
View File

@@ -305,8 +305,8 @@ class Telegram(RPC):
)
stats_tab = tabulate(
[[day['date'],
f"{day['abs_profit']} {stats['stake_currency']}",
f"{day['fiat_value']} {stats['fiat_display_currency']}",
f"{day['abs_profit']:.8f} {stats['stake_currency']}",
f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}",
f"{day['trade_count']} trades"] for day in stats['data']],
headers=[
'Day',

4
freqtrade/strategy/interface.py
View File

@@ -44,6 +44,10 @@ class SellType(Enum):
EMERGENCY_SELL = "emergency_sell"
NONE = ""
def __str__(self):
# explicitly convert to String to help with exporting data.
return self.value
class SellCheckTuple(NamedTuple):
"""

51
freqtrade/templates/strategy_analysis_example.ipynb
View File

@@ -34,7 +34,7 @@
"# config = Configuration.from_files([\"config.json\"])\n",
"\n",
"# Define some constants\n",
"config[\"ticker_interval\"] = \"5m\"\n",
"config[\"timeframe\"] = \"5m\"\n",
"# Name of the strategy class\n",
"config[\"strategy\"] = \"SampleStrategy\"\n",
"# Location of the data\n",
@@ -53,7 +53,7 @@
"from freqtrade.data.history import load_pair_history\n",
"\n",
"candles = load_pair_history(datadir=data_location,\n",
" timeframe=config[\"ticker_interval\"],\n",
" timeframe=config[\"timeframe\"],\n",
" pair=pair)\n",
"\n",
"# Confirm success\n",
@@ -136,10 +136,51 @@
"metadata": {},
"outputs": [],
"source": [
"from freqtrade.data.btanalysis import load_backtest_data\n",
"from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n",
"\n",
"# Load backtest results\n",
"trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n",
"# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n",
"backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
"# backtest_dir can also point to a specific file \n",
"# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# You can get the full backtest statistics by using the following command.\n",
"# This contains all information used to generate the backtest result.\n",
"stats = load_backtest_stats(backtest_dir)\n",
"\n",
"strategy = 'SampleStrategy'\n",
"# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.\n",
"# Example usages:\n",
"print(stats['strategy'][strategy]['results_per_pair'])\n",
"# Get pairlist used for this backtest\n",
"print(stats['strategy'][strategy]['pairlist'])\n",
"# Get market change (average change of all pairs from start to end of the backtest period)\n",
"print(stats['strategy'][strategy]['market_change'])\n",
"# Maximum drawdown ()\n",
"print(stats['strategy'][strategy]['max_drawdown'])\n",
"# Maximum drawdown start and end\n",
"print(stats['strategy'][strategy]['drawdown_start'])\n",
"print(stats['strategy'][strategy]['drawdown_end'])\n",
"\n",
"\n",
"# Get strategy comparison (only relevant if multiple strategies were compared)\n",
"print(stats['strategy_comparison'])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Load backtested trades as dataframe\n",
"trades = load_backtest_data(backtest_dir)\n",
"\n",
"# Show value-counts per pair\n",
"trades.groupby(\"pair\")[\"sell_reason\"].value_counts()"