adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into interface_ordertimeoutcallback

Browse Source
This commit is contained in:
Matthias
2020-04-19 06:58:44 +02:00
parent a1bbeaa668 4bbade245c
commit 431b244f43
57 changed files with 1241 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
freqtrade/__init__.py
View File

@@ -24,4 +24,11 @@ if __version__ == 'develop':
# stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
except Exception:
# git not available, ignore
pass
try:
# Try Fallback to freqtrade_commit file (created by CI while building docker image)
from pathlib import Path
versionfile = Path('./freqtrade_commit')
if versionfile.is_file():
__version__ = f"docker-{versionfile.read_text()[:8]}"
except Exception:
pass

2
freqtrade/commands/arguments.py
View File

@@ -59,7 +59,7 @@ ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchang
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",
"timerange", "ticker_interval"]
"timerange", "ticker_interval", "no_trades"]
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "ticker_interval"]

5
freqtrade/commands/cli_options.py
View File

@@ -413,6 +413,11 @@ AVAILABLE_CLI_OPTIONS = {
metavar='INT',
default=750,
),
"no_trades": Arg(
'--no-trades',
help='Skip using trades from backtesting file and DB.',
action='store_true',
),
"trade_source": Arg(
'--trade-source',
help='Specify the source for trades (Can be DB or file (backtest file)) '

4
freqtrade/commands/hyperopt_commands.py
View File

@@ -52,8 +52,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
if not export_csv:
try:
Hyperopt.print_result_table(config, trials, total_epochs,
not filteroptions['only_best'], print_colorized, 0)
print(Hyperopt.get_result_table(config, trials, total_epochs,
not filteroptions['only_best'], print_colorized, 0))
except KeyboardInterrupt:
print('User interrupted..')

3
freqtrade/configuration/configuration.py
View File

@@ -359,6 +359,9 @@ class Configuration:
self._args_to_config(config, argname='erase',
logstring='Erase detected. Deleting existing data.')
self._args_to_config(config, argname='no_trades',
logstring='Parameter --no-trades detected.')
self._args_to_config(config, argname='timeframes',
logstring='timeframes --timeframes: {}')

33
freqtrade/configuration/load_config.py
View File

@@ -1,13 +1,15 @@
"""
This module contain functions to load the configuration file
"""
import rapidjson
import logging
import re
import sys
from pathlib import Path
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
import rapidjson
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
@@ -15,6 +17,26 @@ logger = logging.getLogger(__name__)
CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS
def log_config_error_range(path: str, errmsg: str) -> str:
"""
Parses configuration file and prints range around error
"""
if path != '-':
offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
if offsetlist:
offset = int(offsetlist[0])
text = Path(path).read_text()
# Fetch an offset of 80 characters around the error line
subtext = text[offset-min(80, offset):offset+80]
segments = subtext.split('\n')
if len(segments) > 3:
# Remove first and last lines, to avoid odd truncations
return '\n'.join(segments[1:-1])
else:
return subtext
return ''
def load_config_file(path: str) -> Dict[str, Any]:
"""
Loads a config file from the given path
@@ -29,5 +51,12 @@ def load_config_file(path: str) -> Dict[str, Any]:
raise OperationalException(
f'Config file "{path}" not found!'
' Please create a config file or check whether it exists.')
except rapidjson.JSONDecodeError as e:
err_range = log_config_error_range(path, str(e))
raise OperationalException(
f'{e}\n'
f'Please verify the following segment of your configuration:\n{err_range}'
if err_range else 'Please verify your configuration file for syntax errors.'
)
return config

33
freqtrade/data/btanalysis.py
View File

@@ -111,7 +111,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
t.calc_profit(), t.calc_profit_ratio(),
t.open_rate, t.close_rate, t.amount,
(round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
if t.close_date else None),
if t.close_date else None),
t.sell_reason,
t.fee_open, t.fee_close,
t.open_rate_requested,
@@ -129,7 +129,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
return trades
def load_trades(source: str, db_url: str, exportfilename: Path) -> pd.DataFrame:
def load_trades(source: str, db_url: str, exportfilename: Path,
no_trades: bool = False) -> pd.DataFrame:
"""
Based on configuration option "trade_source":
* loads data from DB (using `db_url`)
@@ -137,21 +138,33 @@ def load_trades(source: str, db_url: str, exportfilename: Path) -> pd.DataFrame:
:param source: "DB" or "file" - specify source to load from
:param db_url: sqlalchemy formatted url to a database
:param exportfilename: Json file generated by backtesting
:param no_trades: Skip using trades, only return backtesting data columns
:return: DataFrame containing trades
"""
if no_trades:
df = pd.DataFrame(columns=BT_DATA_COLUMNS)
return df
if source == "DB":
return load_trades_from_db(db_url)
elif source == "file":
return load_backtest_data(exportfilename)
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
date_index=False) -> pd.DataFrame:
"""
Compare trades and backtested pair DataFrames to get trades performed on backtested period
:return: the DataFrame of a trades of period
"""
trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
(trades['close_time'] <= dataframe.iloc[-1]['date'])]
if date_index:
trades_start = dataframe.index[0]
trades_stop = dataframe.index[-1]
else:
trades_start = dataframe.iloc[0]['date']
trades_stop = dataframe.iloc[-1]['date']
trades = trades.loc[(trades['open_time'] >= trades_start) &
(trades['close_time'] <= trades_stop)]
return trades
@@ -207,13 +220,15 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time'
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col)
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
high_date = profit_results.loc[max_drawdown_df['high_value'].idxmax(), date_col]
low_date = profit_results.loc[max_drawdown_df['drawdown'].idxmin(), date_col]
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
raise ValueError("No losing trade, therefore no drawdown.")
high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
low_date = profit_results.loc[idxmin, date_col]
return abs(min(max_drawdown_df['drawdown'])), high_date, low_date

12
freqtrade/edge/edge_positioning.py
View File

@@ -8,10 +8,10 @@ import numpy as np
import utils_find_1st as utf1st
from pandas import DataFrame
from freqtrade import constants
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
from freqtrade.exceptions import OperationalException
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.strategy.interface import SellType
logger = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ class Edge:
if self.config['max_open_trades'] != float('inf'):
logger.critical('max_open_trades should be -1 in config !')
if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
if self.config['stake_amount'] != UNLIMITED_STAKE_AMOUNT:
raise OperationalException('Edge works only with unlimited stake amount')
# Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
@@ -96,7 +96,7 @@ class Edge:
logger.info('Using local backtesting data (using whitelist in given config) ...')
if self._refresh_pairs:
history.refresh_data(
refresh_data(
datadir=self.config['datadir'],
pairs=pairs,
exchange=self.exchange,
@@ -104,7 +104,7 @@ class Edge:
timerange=self._timerange,
)
data = history.load_data(
data = load_data(
datadir=self.config['datadir'],
pairs=pairs,
timeframe=self.strategy.ticker_interval,
@@ -122,7 +122,7 @@ class Edge:
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Print timeframe
min_date, max_date = history.get_timerange(preprocessed)
min_date, max_date = get_timerange(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days) ...',
min_date.isoformat(),

50
freqtrade/exchange/exchange.py
View File

@@ -452,6 +452,17 @@ class Exchange:
price = ceil(big_price) / pow(10, symbol_prec)
return price
def price_get_one_pip(self, pair: str, price: float) -> float:
"""
Get's the "1 pip" value for this pair.
Used in PriceFilter to calculate the 1pip movements.
"""
precision = self.markets[pair]['precision']['price']
if self.precisionMode == TICK_SIZE:
return precision
else:
return 1 / pow(10, precision)
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{randint(0, 10**6)}'
@@ -902,6 +913,14 @@ class Exchange:
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
def check_order_canceled_empty(self, order: Dict) -> bool:
"""
Verify if an order has been cancelled without being partially filled
:param order: Order dict as returned from get_order()
:return: True if order has been cancelled without being filled, False otherwise.
"""
return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0
@retrier
def cancel_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
@@ -918,6 +937,37 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def is_cancel_order_result_suitable(self, corder) -> bool:
if not isinstance(corder, dict):
return False
required = ('fee', 'status', 'amount')
return all(k in corder for k in required)
def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
"""
Cancel order returning a result.
Creates a fake result if cancel order returns a non-usable result
and get_order does not work (certain exchanges don't return cancelled orders)
:param order_id: Orderid to cancel
:param pair: Pair corresponding to order_id
:param amount: Amount to use for fake response
:return: Result from either cancel_order if usable, or fetch_order
"""
try:
corder = self.cancel_order(order_id, pair)
if self.is_cancel_order_result_suitable(corder):
return corder
except InvalidOrderException:
logger.warning(f"Could not cancel order {order_id}.")
try:
order = self.get_order(order_id, pair)
except InvalidOrderException:
logger.warning(f"Could not fetch cancelled order {order_id}.")
order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
return order
@retrier
def get_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:

95
freqtrade/freqtradebot.py
View File

@@ -20,6 +20,7 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.exceptions import DependencyException, InvalidOrderException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -144,6 +145,10 @@ class FreqtradeBot:
self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist),
self.strategy.informative_pairs())
with self._sell_lock:
# Check and handle any timed out open orders
self.check_handle_timedout()
# Protect from collisions with forcesell.
# Without this, freqtrade my try to recreate stoploss_on_exchange orders
# while selling is in process, since telegram messages arrive in an different thread.
@@ -155,8 +160,6 @@ class FreqtradeBot:
if self.get_free_open_trades():
self.enter_positions()
# Check and handle any timed out open orders
self.check_handle_timedout()
Trade.session.flush()
def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]:
@@ -395,16 +398,18 @@ class FreqtradeBot:
logger.info(f"Pair {pair} is currently locked.")
return False
# get_free_open_trades is checked before create_trade is called
# but it is still used here to prevent opening too many trades within one iteration
if not self.get_free_open_trades():
logger.debug(f"Can't open a new trade for {pair}: max number of trades is reached.")
return False
# running get_signal on historical data fetched
(buy, sell) = self.strategy.get_signal(
pair, self.strategy.ticker_interval,
self.dataprovider.ohlcv(pair, self.strategy.ticker_interval))
if buy and not sell:
if not self.get_free_open_trades():
logger.debug("Can't open a new trade: max number of trades is reached.")
return False
stake_amount = self.get_trade_stake_amount(pair)
if not stake_amount:
logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
@@ -599,7 +604,6 @@ class FreqtradeBot:
trades_closed = 0
for trade in trades:
try:
self.update_trade_state(trade)
if (self.strategy.order_types.get('stoploss_on_exchange') and
self.handle_stoploss_on_exchange(trade)):
@@ -859,19 +863,13 @@ class FreqtradeBot:
continue
order = self.exchange.get_order(trade.open_order_id, trade.pair)
except (RequestException, DependencyException, InvalidOrderException):
logger.info(
'Cannot query order for %s due to %s',
trade,
traceback.format_exc())
logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
continue
# Check if trade is still actually open
if float(order.get('remaining', 0.0)) == 0.0:
self.wallets.update()
continue
trade_state_update = self.update_trade_state(trade, order)
if (order['side'] == 'buy' and (
order['status'] == 'canceled'
trade_state_update
or self._check_timed_out('buy', order)
or strategy_safe_wrapper(self.strategy.check_buy_timeout,
default_retval=False)(pair=trade.pair,
@@ -884,16 +882,16 @@ class FreqtradeBot:
self._notify_buy_cancel(trade, order_type)
elif (order['side'] == 'sell' and (
order['status'] == 'canceled'
trade_state_update
or self._check_timed_out('sell', order)
or strategy_safe_wrapper(self.strategy.check_sell_timeout,
default_retval=False)(pair=trade.pair,
trade=trade,
order=order))):
self.handle_timedout_limit_sell(trade, order)
reason = self.handle_timedout_limit_sell(trade, order)
self.wallets.update()
order_type = self.strategy.order_types['sell']
self._notify_sell_cancel(trade, order_type)
self._notify_sell_cancel(trade, order_type, reason)
def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool:
"""
@@ -902,15 +900,17 @@ class FreqtradeBot:
"""
if order['status'] != 'canceled':
reason = "cancelled due to timeout"
corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
logger.info('Buy order %s for %s.', reason, trade)
corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
else:
# Order was cancelled already, so we can reuse the existing dict
corder = order
reason = "cancelled on exchange"
logger.info('Buy order %s for %s.', reason, trade)
if corder.get('remaining', order['remaining']) == order['amount']:
logger.info('Buy order %s for %s.', reason, trade)
if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
logger.info('Buy order fully cancelled. Removing %s from database.', trade)
# if trade is not partially completed, just delete the trade
Trade.session.delete(trade)
Trade.session.flush()
@@ -921,19 +921,10 @@ class FreqtradeBot:
# cancel_order may not contain the full order dict, so we need to fallback
# to the order dict aquired before cancelling.
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = order['amount'] - corder.get('remaining', order['remaining'])
trade.amount = order['amount'] - safe_value_fallback(corder, order,
'remaining', 'remaining')
trade.stake_amount = trade.amount * trade.open_rate
# verify if fees were taken from amount to avoid problems during selling
try:
new_amount = self.get_real_amount(trade, corder if 'fee' in corder else order,
trade.amount)
if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
trade.amount = new_amount
# Fee was applied, so set to 0
trade.fee_open = 0
trade.recalc_open_trade_price()
except DependencyException as e:
logger.warning("Could not update trade amount: %s", e)
self.update_trade_state(trade, corder, trade.amount)
trade.open_order_id = None
logger.info('Partial buy order timeout for %s.', trade)
@@ -943,14 +934,14 @@ class FreqtradeBot:
})
return False
def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool:
def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> str:
"""
Sell timeout - cancel order and update trade
:return: True if order was fully cancelled
:return: Reason for cancel
"""
# if trade is not partially completed, just cancel the trade
if order['remaining'] == order['amount']:
if order["status"] != "canceled":
if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
if not self.exchange.check_order_canceled_empty(order):
reason = "cancelled due to timeout"
# if trade is not partially completed, just delete the trade
self.exchange.cancel_order(trade.open_order_id, trade.pair)
@@ -960,15 +951,17 @@ class FreqtradeBot:
logger.info('Sell order %s for %s.', reason, trade)
trade.close_rate = None
trade.close_rate_requested = None
trade.close_profit = None
trade.close_profit_abs = None
trade.close_date = None
trade.is_open = True
trade.open_order_id = None
return True
return reason
# TODO: figure out how to handle partially complete sell orders
return False
return 'partially filled - keeping order open'
def _safe_sell_amount(self, pair: str, amount: float) -> float:
"""
@@ -1087,7 +1080,7 @@ class FreqtradeBot:
# Send the message
self.rpc.send_msg(msg)
def _notify_sell_cancel(self, trade: Trade, order_type: str) -> None:
def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
"""
Sends rpc notification when a sell cancel occured.
"""
@@ -1114,6 +1107,7 @@ class FreqtradeBot:
'close_date': trade.close_date,
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency', None),
'reason': reason,
}
if 'fiat_display_currency' in self.config:
@@ -1128,9 +1122,12 @@ class FreqtradeBot:
# Common update trade state methods
#
def update_trade_state(self, trade: Trade, action_order: dict = None) -> None:
def update_trade_state(self, trade: Trade, action_order: dict = None,
order_amount: float = None) -> bool:
"""
Checks trades with open orders and updates the amount if necessary
Handles closing both buy and sell orders.
:return: True if order has been cancelled without being filled partially, False otherwise
"""
# Get order details for actual price per unit
if trade.open_order_id:
@@ -1140,25 +1137,31 @@ class FreqtradeBot:
order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair)
except InvalidOrderException as exception:
logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception)
return
return False
# Try update amount (binance-fix)
try:
new_amount = self.get_real_amount(trade, order)
new_amount = self.get_real_amount(trade, order, order_amount)
if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
order['amount'] = new_amount
order.pop('filled', None)
# Fee was applied, so set to 0
trade.fee_open = 0
trade.recalc_open_trade_price()
except DependencyException as exception:
logger.warning("Could not update trade amount: %s", exception)
if self.exchange.check_order_canceled_empty(order):
# Trade has been cancelled on exchange
# Handling of this will happen in check_handle_timeout.
return True
trade.update(order)
# Updating wallets when order is closed
if not trade.is_open:
self.wallets.update()
return False
def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
"""
Get real amount for the trade

6
freqtrade/loggers.py
View File

@@ -18,13 +18,13 @@ def _set_loggers(verbosity: int = 0) -> None:
"""
logging.getLogger('requests').setLevel(
logging.INFO if verbosity <= 1 else logging.DEBUG
logging.INFO if verbosity <= 1 else logging.DEBUG
)
logging.getLogger("urllib3").setLevel(
logging.INFO if verbosity <= 1 else logging.DEBUG
logging.INFO if verbosity <= 1 else logging.DEBUG
)
logging.getLogger('ccxt.base.exchange').setLevel(
logging.INFO if verbosity <= 2 else logging.DEBUG
logging.INFO if verbosity <= 2 else logging.DEBUG
)
logging.getLogger('telegram').setLevel(logging.INFO)

15
freqtrade/misc.py
View File

@@ -134,6 +134,21 @@ def round_dict(d, n):
return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
"""
Search a value in dict1, return this if it's not None.
Fall back to dict2 - return key2 from dict2 if it's not None.
Else falls back to None.
"""
if key1 in dict1 and dict1[key1] is not None:
return dict1[key1]
else:
if key2 in dict2 and dict2[key2] is not None:
return dict2[key2]
return default_value
def plural(num: float, singular: str, plural: str = None) -> str:
return singular if (num == 1 or num == -1) else plural or singular + 's'

77
freqtrade/optimize/backtesting.py
View File

@@ -6,8 +6,7 @@ This module contains the backtesting logic
import logging
from copy import deepcopy
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
import arrow
from pandas import DataFrame
@@ -19,10 +18,8 @@ from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.misc import file_dump_json
from freqtrade.optimize.optimize_reports import (
generate_text_table, generate_text_table_sell_reason,
generate_text_table_strategy)
from freqtrade.optimize.optimize_reports import (show_backtest_results,
store_backtest_result)
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
@@ -108,7 +105,7 @@ class Backtesting:
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
def load_bt_data(self):
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
@@ -134,23 +131,6 @@ class Backtesting:
return data, timerange
def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
strategyname: Optional[str] = None) -> None:
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
for index, t in results.iterrows()]
if records:
if strategyname:
# Inject strategyname to filename
recordfilename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
logger.info(f'Dumping backtest results to {recordfilename}')
file_dump_json(recordfilename, records)
def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
"""
Helper function to convert a processed dataframes into lists for performance reasons.
@@ -169,8 +149,8 @@ class Backtesting:
# To avoid using data from future, we use buy/sell signals shifted
# from the previous candle
df_analyzed.loc[:, 'buy'] = df_analyzed['buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed['sell'].shift(1)
df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
@@ -418,44 +398,7 @@ class Backtesting:
position_stacking=position_stacking,
)
for strategy, results in all_results.items():
if self.config.get('export', False):
self._store_backtest_result(self.config['exportfilename'], results,
strategy if len(self.strategylist) > 1 else None)
print(f"Result for strategy {strategy}")
table = generate_text_table(data, stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = generate_text_table_sell_reason(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results.loc[results.open_at_end], skip_nan=True)
if isinstance(table, str):
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str):
print('=' * len(table.splitlines()[0]))
print()
if len(all_results) > 1:
# Print Strategy summary table
table = generate_text_table_strategy(self.config['stake_currency'],
self.config['max_open_trades'],
all_results=all_results)
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
print(table)
print('=' * len(table.splitlines()[0]))
print('\nFor more details, please look at the detail tables above')
if self.config.get('export', False):
store_backtest_result(self.config['exportfilename'], all_results)
# Show backtest results
show_backtest_results(self.config, data, all_results)

141
freqtrade/optimize/hyperopt.py
View File

@@ -7,7 +7,6 @@ This module contains the hyperopt logic
import locale
import logging
import random
import sys
import warnings
from math import ceil
from collections import OrderedDict
@@ -18,10 +17,10 @@ from typing import Any, Dict, List, Optional
import rapidjson
from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import (Parallel, cpu_count, delayed, dump, load,
wrap_non_picklable_objects)
from pandas import DataFrame, json_normalize, isna
import progressbar
import tabulate
from os import path
import io
@@ -43,7 +42,8 @@ with warnings.catch_warnings():
from skopt import Optimizer
from skopt.space import Dimension
progressbar.streams.wrap_stderr()
progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__)
@@ -266,21 +266,33 @@ class Hyperopt:
Log results if it is better than any previous evaluation
"""
is_best = results['is_best']
if not self.print_all:
# Print '\n' after each 100th epoch to separate dots from the log messages.
# Otherwise output is messy on a terminal.
print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore
sys.stdout.flush()
if self.print_all or is_best:
if not self.print_all:
# Separate the results explanation string from dots
print("\n")
self.print_result_table(self.config, results, self.total_epochs,
self.print_all, self.print_colorized,
self.hyperopt_table_header)
print(
self.get_result_table(
self.config, results, self.total_epochs,
self.print_all, self.print_colorized,
self.hyperopt_table_header
)
)
self.hyperopt_table_header = 2
def get_results(self, results) -> str:
"""
Log results if it is better than any previous evaluation
"""
output = ''
is_best = results['is_best']
if self.print_all or is_best:
output = self.get_result_table(
self.config, results, self.total_epochs,
self.print_all, self.print_colorized,
self.hyperopt_table_header
)
self.hyperopt_table_header = 2
return output
@staticmethod
def print_results_explanation(results, total_epochs, highlight_best: bool,
print_colorized: bool) -> None:
@@ -304,13 +316,13 @@ class Hyperopt:
f"Objective: {results['loss']:.5f}")
@staticmethod
def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
print_colorized: bool, remove_header: int) -> None:
def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
print_colorized: bool, remove_header: int) -> str:
"""
Log result table
"""
if not results:
return
return ''
tabulate.PRESERVE_WHITESPACE = True
@@ -381,7 +393,7 @@ class Hyperopt:
trials.to_dict(orient='list'), tablefmt='psql',
headers='keys', stralign="right"
)
print(table)
return table
@staticmethod
def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
@@ -653,48 +665,75 @@ class Hyperopt:
self.dimensions: List[Dimension] = self.hyperopt_space()
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.print_colorized:
colorama_init(autoreset=True)
try:
with Parallel(n_jobs=config_jobs) as parallel:
jobs = parallel._effective_n_jobs()
logger.info(f'Effective number of parallel workers used: {jobs}')
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
# iteration (should not exceed self.total_epochs in total)
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
asked = self.opt.ask(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
self.fix_optimizer_models_list()
# Define progressbar
if self.print_colorized:
widgets = [
' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
' (', progressbar.Percentage(), ')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}',
fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
)),
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
else:
widgets = [
' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
' (', progressbar.Percentage(), ')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}',
)),
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
with progressbar.ProgressBar(
maxval=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
widgets=widgets
) as pbar:
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
# iteration (should not exceed self.total_epochs in total)
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
for j, val in enumerate(f_val):
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
asked = self.opt.ask(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
self.fix_optimizer_models_list()
is_best = self.is_best_loss(val, self.current_best_loss)
# This value is assigned here and not in the optimization method
# to keep proper order in the list of results. That's because
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
# Calculate progressbar outputs
for j, val in enumerate(f_val):
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
self.print_results(val)
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = self.is_best_loss(val, self.current_best_loss)
# This value is assigned here and not in the optimization method
# to keep proper order in the list of results. That's because
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
self.print_results(val)
if is_best:
self.current_best_loss = val['loss']
self.trials.append(val)
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self.save_trials()
pbar.update(current)
if is_best:
self.current_best_loss = val['loss']
self.trials.append(val)
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self.save_trials()
except KeyboardInterrupt:
print('User interrupted..')

77
freqtrade/optimize/optimize_reports.py
View File

@@ -1,9 +1,38 @@
import logging
from datetime import timedelta
from pathlib import Path
from typing import Dict
from pandas import DataFrame
from tabulate import tabulate
from freqtrade.misc import file_dump_json
logger = logging.getLogger(__name__)
def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
"""
Stores backtest results to file (one file per strategy)
:param recordfilename: Destination filename
:param all_results: Dict of Dataframes, one results dataframe per strategy
"""
for strategy, results in all_results.items():
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
for index, t in results.iterrows()]
if records:
filename = recordfilename
if len(all_results) > 1:
# Inject strategy to filename
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
logger.info(f'Dumping backtest results to {filename}')
file_dump_json(filename, records)
def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
results: DataFrame, skip_nan: bool = False) -> str:
@@ -69,12 +98,12 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def generate_text_table_sell_reason(
data: Dict[str, Dict], stake_currency: str, max_open_trades: int, results: DataFrame
) -> str:
def generate_text_table_sell_reason(stake_currency: str, max_open_trades: int,
results: DataFrame) -> str:
"""
Generate small table outlining Backtest results
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
:param stake_currency: Stakecurrency used
:param max_open_trades: Max_open_trades parameter
:param results: Dataframe containing the backtest results
:return: pretty printed table with tabulate as string
"""
@@ -173,3 +202,43 @@ def generate_edge_table(results: dict) -> str:
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame],
all_results: Dict[str, DataFrame]):
for strategy, results in all_results.items():
print(f"Result for strategy {strategy}")
table = generate_text_table(btdata, stake_currency=config['stake_currency'],
max_open_trades=config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = generate_text_table_sell_reason(stake_currency=config['stake_currency'],
max_open_trades=config['max_open_trades'],
results=results)
if isinstance(table, str):
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = generate_text_table(btdata,
stake_currency=config['stake_currency'],
max_open_trades=config['max_open_trades'],
results=results.loc[results.open_at_end], skip_nan=True)
if isinstance(table, str):
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str):
print('=' * len(table.splitlines()[0]))
print()
if len(all_results) > 1:
# Print Strategy summary table
table = generate_text_table_strategy(config['stake_currency'],
config['max_open_trades'],
all_results=all_results)
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
print(table)
print('=' * len(table.splitlines()[0]))
print('\nFor more details, please look at the detail tables above')

23
freqtrade/pairlist/IPairList.py
View File

@@ -9,6 +9,8 @@ from abc import ABC, abstractmethod, abstractproperty
from copy import deepcopy
from typing import Any, Dict, List
from cachetools import TTLCache, cached
from freqtrade.exchange import market_is_active
logger = logging.getLogger(__name__)
@@ -31,6 +33,9 @@ class IPairList(ABC):
self._config = config
self._pairlistconfig = pairlistconfig
self._pairlist_pos = pairlist_pos
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._last_refresh = 0
self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period)
@property
def name(self) -> str:
@@ -40,6 +45,24 @@ class IPairList(ABC):
"""
return self.__class__.__name__
def log_on_refresh(self, logmethod, message: str) -> None:
"""
Logs message - not more often than "refresh_period" to avoid log spamming
Logs the log-message as debug as well to simplify debugging.
:param logmethod: Function that'll be called. Most likely `logger.info`.
:param message: String containing the message to be sent to the function.
:return: None.
"""
@cached(cache=self._log_cache)
def _log_on_refresh(message: str):
logmethod(message)
# Log as debug first
logger.debug(message)
# Call hidden function.
_log_on_refresh(message)
@abstractproperty
def needstickers(self) -> bool:
"""

5
freqtrade/pairlist/PrecisionFilter.py
View File

@@ -39,8 +39,9 @@ class PrecisionFilter(IPairList):
stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99)
logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
if sp <= stop_gap_price:
logger.info(f"Removed {ticker['symbol']} from whitelist, "
f"because stop price {sp} would be <= stop limit {stop_gap_price}")
self.log_on_refresh(logger.info,
f"Removed {ticker['symbol']} from whitelist, "
f"because stop price {sp} would be <= stop limit {stop_gap_price}")
return False
return True

10
freqtrade/pairlist/PriceFilter.py
View File

@@ -35,16 +35,14 @@ class PriceFilter(IPairList):
"""
Check if if one price-step (pip) is > than a certain barrier.
:param ticker: ticker dict as returned from ccxt.load_markets()
:param precision: Precision
:return: True if the pair can stay, false if it should be removed
"""
precision = self._exchange.markets[ticker['symbol']]['precision']['price']
compare = ticker['last'] + 1 / pow(10, precision)
compare = ticker['last'] + self._exchange.price_get_one_pip(ticker['symbol'],
ticker['last'])
changeperc = (compare - ticker['last']) / ticker['last']
if changeperc > self._low_price_ratio:
logger.info(f"Removed {ticker['symbol']} from whitelist, "
f"because 1 unit is {changeperc * 100:.3f}%")
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because 1 unit is {changeperc * 100:.3f}%")
return False
return True

6
freqtrade/pairlist/SpreadFilter.py
View File

@@ -49,9 +49,9 @@ class SpreadFilter(IPairList):
if 'bid' in ticker and 'ask' in ticker:
spread = 1 - ticker['bid'] / ticker['ask']
if not ticker or spread > self._max_spread_ratio:
logger.info(f"Removed {ticker['symbol']} from whitelist, "
f"because spread {spread * 100:.3f}% >"
f"{self._max_spread_ratio * 100}%")
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because spread {spread * 100:.3f}% >"
f"{self._max_spread_ratio * 100}%")
pairlist.remove(p)
else:
pairlist.remove(p)

21
freqtrade/pairlist/VolumePairList.py
View File

@@ -39,7 +39,6 @@ class VolumePairList(IPairList):
if not self._validate_keys(self._sort_key):
raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}')
self._last_refresh = 0
@property
def needstickers(self) -> bool:
@@ -68,16 +67,18 @@ class VolumePairList(IPairList):
:return: new whitelist
"""
# Generate dynamic whitelist
if self._last_refresh + self.refresh_period < datetime.now().timestamp():
# Must always run if this pairlist is not the first in the list.
if (self._pairlist_pos != 0 or
(self._last_refresh + self.refresh_period < datetime.now().timestamp())):
self._last_refresh = int(datetime.now().timestamp())
return self._gen_pair_whitelist(pairlist,
tickers,
self._config['stake_currency'],
self._sort_key,
self._min_value
)
pairs = self._gen_pair_whitelist(pairlist, tickers,
self._config['stake_currency'],
self._sort_key, self._min_value)
else:
return pairlist
pairs = pairlist
self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
return pairs
def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict,
base_currency: str, key: str, min_val: int) -> List[str]:
@@ -88,7 +89,6 @@ class VolumePairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers()).
:return: List of pairs
"""
if self._pairlist_pos == 0:
# If VolumePairList is the first in the list, use fresh pairlist
# Check if pair quote currency equals to the stake currency.
@@ -109,6 +109,5 @@ class VolumePairList(IPairList):
pairs = self._verify_blacklist(pairs, aswarning=False)
# Limit to X number of pairs
pairs = pairs[:self._number_pairs]
logger.info(f"Searching {self._number_pairs} pairs: {pairs}")
return pairs

28
freqtrade/persistence.py
View File

@@ -86,7 +86,7 @@ def check_migrate(engine) -> None:
logger.debug(f'trying {table_back_name}')
# Check for latest column
if not has_column(cols, 'open_trade_price'):
if not has_column(cols, 'close_profit_abs'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
@@ -106,6 +106,9 @@ def check_migrate(engine) -> None:
ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
@@ -123,7 +126,7 @@ def check_migrate(engine) -> None:
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, strategy,
ticker_interval, open_trade_price
ticker_interval, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
@@ -143,7 +146,7 @@ def check_migrate(engine) -> None:
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{strategy} strategy, {ticker_interval} ticker_interval,
{open_trade_price} open_trade_price
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
@@ -185,11 +188,12 @@ class Trade(_DECL_BASE):
fee_close = Column(Float, nullable=False, default=0.0)
open_rate = Column(Float)
open_rate_requested = Column(Float)
# open_trade_price - calcuated via _calc_open_trade_price
# open_trade_price - calculated via _calc_open_trade_price
open_trade_price = Column(Float)
close_rate = Column(Float)
close_rate_requested = Column(Float)
close_profit = Column(Float)
close_profit_abs = Column(Float)
stake_amount = Column(Float, nullable=False)
amount = Column(Float)
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
@@ -229,6 +233,9 @@ class Trade(_DECL_BASE):
return {
'trade_id': self.id,
'pair': self.pair,
'is_open': self.is_open,
'fee_open': self.fee_open,
'fee_close': self.fee_close,
'open_date_hum': arrow.get(self.open_date).humanize(),
'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
'close_date_hum': (arrow.get(self.close_date).humanize()
@@ -236,14 +243,24 @@ class Trade(_DECL_BASE):
'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
if self.close_date else None),
'open_rate': self.open_rate,
'open_rate_requested': self.open_rate_requested,
'open_trade_price': self.open_trade_price,
'close_rate': self.close_rate,
'close_rate_requested': self.close_rate_requested,
'amount': round(self.amount, 8),
'stake_amount': round(self.stake_amount, 8),
'close_profit': self.close_profit,
'sell_reason': self.sell_reason,
'stop_loss': self.stop_loss,
'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
'initial_stop_loss': self.initial_stop_loss,
'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100
if self.initial_stop_loss_pct else None),
'min_rate': self.min_rate,
'max_rate': self.max_rate,
'strategy': self.strategy,
'ticker_interval': self.ticker_interval,
'open_order_id': self.open_order_id,
}
def adjust_min_max_rates(self, current_price: float) -> None:
@@ -311,7 +328,7 @@ class Trade(_DECL_BASE):
if order_type in ('market', 'limit') and order['side'] == 'buy':
# Update open rate and actual amount
self.open_rate = Decimal(order['price'])
self.amount = Decimal(order['amount'])
self.amount = Decimal(order.get('filled', order['amount']))
self.recalc_open_trade_price()
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
self.open_order_id = None
@@ -334,6 +351,7 @@ class Trade(_DECL_BASE):
"""
self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_ratio()
self.close_profit_abs = self.calc_profit()
self.close_date = datetime.utcnow()
self.is_open = False
self.open_order_id = None

35
freqtrade/plot/plotting.py
View File

@@ -10,6 +10,7 @@ from freqtrade.data.btanalysis import (calculate_max_drawdown,
create_cum_profit,
extract_trades_of_period, load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.exchange import timeframe_to_prev_date
from freqtrade.data.history import load_data
from freqtrade.misc import pair_to_filename
from freqtrade.resolvers import StrategyResolver
@@ -48,11 +49,21 @@ def init_plotscript(config):
data_format=config.get('dataformat_ohlcv', 'json'),
)
trades = load_trades(config['trade_source'],
db_url=config.get('db_url'),
exportfilename=config.get('exportfilename'),
)
no_trades = False
if config.get('no_trades', False):
no_trades = True
elif not config['exportfilename'].is_file() and config['trade_source'] == 'file':
logger.warning("Backtest file is missing skipping trades.")
no_trades = True
trades = load_trades(
config['trade_source'],
db_url=config.get('db_url'),
exportfilename=config.get('exportfilename'),
no_trades=no_trades
)
trades = trim_dataframe(trades, timerange, 'open_time')
return {"ohlcv": data,
"trades": trades,
"pairs": pairs,
@@ -112,7 +123,8 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub
return fig
def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> make_subplots:
def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
timeframe: str) -> make_subplots:
"""
Add scatter points indicating max drawdown
"""
@@ -122,12 +134,12 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> m
drawdown = go.Scatter(
x=[highdate, lowdate],
y=[
df_comb.loc[highdate, 'cum_profit'],
df_comb.loc[lowdate, 'cum_profit'],
df_comb.loc[timeframe_to_prev_date(timeframe, highdate), 'cum_profit'],
df_comb.loc[timeframe_to_prev_date(timeframe, lowdate), 'cum_profit'],
],
mode='markers',
name=f"Max drawdown {max_drawdown:.2f}%",
text=f"Max drawdown {max_drawdown:.2f}%",
name=f"Max drawdown {max_drawdown * 100:.2f}%",
text=f"Max drawdown {max_drawdown * 100:.2f}%",
marker=dict(
symbol='square-open',
size=9,
@@ -373,6 +385,9 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
# Combine close-values for all pairs, rename columns to "pair"
df_comb = combine_dataframes_with_mean(data, "close")
# Trim trades to available OHLCV data
trades = extract_trades_of_period(df_comb, trades, date_index=True)
# Add combined cumulative profit
df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe)
@@ -395,7 +410,7 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
fig.add_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
fig = add_max_drawdown(fig, 2, trades, df_comb)
fig = add_max_drawdown(fig, 2, trades, df_comb, timeframe)
for pair in pairs:
profit_col = f'cum_profit_{pair}'

15
freqtrade/rpc/api_server.py
View File

@@ -173,7 +173,8 @@ class ApiServer(RPC):
view_func=self._show_config, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/ping', 'ping',
view_func=self._ping, methods=['GET'])
self.app.add_url_rule(f'{BASE_URI}/trades', 'trades',
view_func=self._trades, methods=['GET'])
# Combined actions and infos
self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist,
methods=['GET', 'POST'])
@@ -358,6 +359,18 @@ class ApiServer(RPC):
self._config.get('fiat_display_currency', ''))
return self.rest_dump(results)
@require_login
@rpc_catch_errors
def _trades(self):
"""
Handler for /trades.
Returns the X last trades in json format
"""
limit = int(request.args.get('limit', 0))
results = self._rpc_trade_history(limit)
return self.rest_dump(results)
@require_login
@rpc_catch_errors
def _whitelist(self):

20
freqtrade/rpc/rpc.py
View File

@@ -197,7 +197,7 @@ class RPC:
Trade.close_date >= profitday,
Trade.close_date < (profitday + timedelta(days=1))
]).order_by(Trade.close_date).all()
curdayprofit = sum(trade.calc_profit() for trade in trades)
curdayprofit = sum(trade.close_profit_abs for trade in trades)
profit_days[profitday] = {
'amount': f'{curdayprofit:.8f}',
'trades': len(trades)
@@ -226,6 +226,20 @@ class RPC:
for key, value in profit_days.items()
]
def _rpc_trade_history(self, limit: int) -> Dict:
""" Returns the X last trades """
if limit > 0:
trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit)
else:
trades = Trade.get_trades().order_by(Trade.id.desc()).all()
output = [trade.to_json() for trade in trades]
return {
"trades": output,
"trades_count": len(output)
}
def _rpc_trade_statistics(
self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
""" Returns cumulative profit statistics """
@@ -246,8 +260,8 @@ class RPC:
durations.append((trade.close_date - trade.open_date).total_seconds())
if not trade.is_open:
profit_ratio = trade.calc_profit_ratio()
profit_closed_coin.append(trade.calc_profit())
profit_ratio = trade.close_profit
profit_closed_coin.append(trade.close_profit_abs)
profit_closed_ratio.append(profit_ratio)
else:
# Get current rate

3
freqtrade/rpc/telegram.py
View File

@@ -172,7 +172,8 @@ class Telegram(RPC):
' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
message = "*{exchange}:* Cancelling Open Sell Order for {pair}".format(**msg)
message = ("*{exchange}:* Cancelling Open Sell Order "
"for {pair}. Reason: {reason}").format(**msg)
elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION:
message = '*Status:* `{status}`'.format(**msg)

31
freqtrade/strategy/interface.py
View File

@@ -278,8 +278,25 @@ class IStrategy(ABC):
return dataframe
def get_signal(self, pair: str, interval: str,
dataframe: DataFrame) -> Tuple[bool, bool]:
@staticmethod
def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:
""" keep some data for dataframes """
return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
@staticmethod
def assert_df(dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
""" make sure data is unmodified """
message = ""
if df_len != len(dataframe):
message = "length"
elif df_close != dataframe["close"].iloc[-1]:
message = "last close price"
elif df_date != dataframe["date"].iloc[-1]:
message = "last date"
if message:
raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC
@@ -291,10 +308,13 @@ class IStrategy(ABC):
logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return False, False
latest_date = dataframe['date'].max()
try:
df_len, df_close, df_date = self.preserve_df(dataframe)
dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair})
self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error:
logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
@@ -304,7 +324,7 @@ class IStrategy(ABC):
logger.warning('Empty dataframe for pair %s', pair)
return False, False
latest = dataframe.iloc[-1]
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
# Check if dataframe is out of date
signal_date = arrow.get(latest['date'])
@@ -473,8 +493,11 @@ class IStrategy(ABC):
"""
Creates a dataframe and populates indicators for given candle (OHLCV) data
Used by optimize operations only, not during dry / live runs.
Using .copy() to get a fresh copy of the dataframe for every strategy run.
Has positive effects on memory usage for whatever reason - also when
using only one strategy.
"""
return {pair: self.advise_indicators(pair_data, {'pair': pair})
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
for pair, pair_data in data.items()}
def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: