adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into safe_sell_amount

Browse Source
This commit is contained in:
Matthias
2019-12-18 19:45:31 +01:00
parent 04257d8ecc ce190a7485
commit 834a0ed620
45 changed files with 812 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
freqtrade/constants.py
View File

@@ -18,7 +18,7 @@ REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter']
DRY_RUN_WALLET = 999.9
DRY_RUN_WALLET = 1000
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons
USERPATH_HYPEROPTS = 'hyperopts'
@@ -75,7 +75,7 @@ CONF_SCHEMA = {
},
'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT},
'dry_run': {'type': 'boolean'},
'dry_run_wallet': {'type': 'number'},
'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET},
'process_only_new_candles': {'type': 'boolean'},
'minimal_roi': {
'type': 'object',
@@ -275,6 +275,7 @@ CONF_SCHEMA = {
'stake_currency',
'stake_amount',
'dry_run',
'dry_run_wallet',
'bid_strategy',
'unfilledtimeout',
'stoploss',

2
freqtrade/data/btanalysis.py
View File

@@ -108,7 +108,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
trades = pd.DataFrame([(t.pair,
t.open_date.replace(tzinfo=timezone.utc),
t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
t.calc_profit(), t.calc_profit_percent(),
t.calc_profit(), t.calc_profit_ratio(),
t.open_rate, t.close_rate, t.amount,
(round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
if t.close_date else None),

131
freqtrade/data/history.py
View File

@@ -68,7 +68,7 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date
def load_tickerdata_file(datadir: Path, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None) -> Optional[list]:
timerange: Optional[TimeRange] = None) -> List[Dict]:
"""
Load a pair from file, either .json.gz or .json
:return: tickerlist or None if unsuccessful
@@ -128,39 +128,26 @@ def load_pair_history(pair: str,
timeframe: str,
datadir: Path,
timerange: Optional[TimeRange] = None,
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
fill_up_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
) -> DataFrame:
"""
Loads cached ticker history for the given pair.
Load cached ticker history for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param datadir: Path to the data storage location.
:param timerange: Limit data to be loaded to this timerange
:param refresh_pairs: Refresh pairs from exchange.
(Note: Requires exchange to be passed as well.)
:param exchange: Exchange object (needed when using "refresh_pairs")
:param fill_up_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:return: DataFrame with ohlcv data, or empty DataFrame
"""
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
# The user forced the refresh of pairs
if refresh_pairs:
download_pair_history(datadir=datadir,
exchange=exchange,
pair=pair,
timeframe=timeframe,
timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
if pairdata:
@@ -180,30 +167,22 @@ def load_pair_history(pair: str,
def load_data(datadir: Path,
timeframe: str,
pairs: List[str],
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True,
startup_candles: int = 0,
fail_without_data: bool = False
) -> Dict[str, DataFrame]:
"""
Loads ticker history data for a list of pairs
Load ticker history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param refresh_pairs: Refresh pairs from exchange.
(Note: Requires exchange to be passed as well.)
:param exchange: Exchange object (needed when using "refresh_pairs")
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
:param startup_candles: Additional candles to load at the start of the period
:param fail_without_data: Raise OperationalException if no data is found.
:return: dict(<pair>:<Dataframe>)
TODO: refresh_pairs is still used by edge to keep the data uptodate.
This should be replaced in the future. Instead, writing the current candles to disk
from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
exchange and refresh_pairs are then not needed here nor in load_pair_history.
"""
result: Dict[str, DataFrame] = {}
if startup_candles > 0 and timerange:
@@ -212,8 +191,6 @@ def load_data(datadir: Path,
for pair in pairs:
hist = load_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing,
startup_candles=startup_candles)
if not hist.empty:
@@ -224,6 +201,27 @@ def load_data(datadir: Path,
return result
def refresh_data(datadir: Path,
timeframe: str,
pairs: List[str],
exchange: Exchange,
timerange: Optional[TimeRange] = None,
) -> None:
"""
Refresh ticker history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param exchange: Exchange object
:param timerange: Limit data to be loaded to this timerange
"""
for pair in pairs:
_download_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
exchange=exchange)
def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path:
pair_s = pair.replace("/", "_")
filename = datadir.joinpath(f'{pair_s}-{timeframe}.json')
@@ -277,11 +275,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
return (data, since_ms)
def download_pair_history(datadir: Path,
exchange: Optional[Exchange],
pair: str,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
def _download_pair_history(datadir: Path,
exchange: Exchange,
pair: str,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@@ -295,11 +293,6 @@ def download_pair_history(datadir: Path,
:param timerange: range of time to download
:return: bool with success state
"""
if not exchange:
raise OperationalException(
"Exchange needs to be initialized when downloading pair history data"
)
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
@@ -312,11 +305,12 @@ def download_pair_history(datadir: Path,
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms if since_ms
else
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000)
days=-30).float_timestamp) * 1000
)
data.extend(new_data)
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
@@ -334,12 +328,12 @@ def download_pair_history(datadir: Path,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: Optional[TimeRange] = None,
datadir: Path, timerange: Optional[TimeRange] = None,
erase=False) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data
:return: Pairs not available
Used by freqtrade download-data subcommand.
:return: List of pairs that are not available.
"""
pairs_not_available = []
for pair in pairs:
@@ -349,23 +343,23 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
continue
for timeframe in timeframes:
dl_file = pair_data_filename(dl_path, pair, timeframe)
dl_file = pair_data_filename(datadir, pair, timeframe)
if erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {timeframe}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, timeframe=str(timeframe),
timerange=timerange)
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
timerange=timerange)
return pairs_not_available
def download_trades_history(datadir: Path,
exchange: Exchange,
pair: str,
timerange: Optional[TimeRange] = None) -> bool:
def _download_trades_history(datadir: Path,
exchange: Exchange,
pair: str,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download trade history from the exchange.
Appends to previously downloaded trades data.
@@ -381,11 +375,11 @@ def download_trades_history(datadir: Path,
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000,
# until=xxx,
from_id=from_id,
)
trades.extend(new_trades[1])
@@ -407,9 +401,9 @@ def download_trades_history(datadir: Path,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]:
"""
Refresh stored trades data.
Used by freqtrade download-data
:return: Pairs not available
Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
:return: List of pairs that are not available.
"""
pairs_not_available = []
for pair in pairs:
@@ -425,9 +419,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
dl_file.unlink()
logger.info(f'Downloading trades for pair {pair}.')
download_trades_history(datadir=datadir, exchange=exchange,
pair=pair,
timerange=timerange)
_download_trades_history(datadir=datadir, exchange=exchange,
pair=pair,
timerange=timerange)
return pairs_not_available
@@ -448,22 +442,23 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data
Get the maximum common timerange for the given backtest data.
:param data: dictionary with preprocessed backtesting data
:return: tuple containing min_date, max_date
"""
timeframe = [
timeranges = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
for frame in data.values()
]
return min(timeframe, key=operator.itemgetter(0))[0], \
max(timeframe, key=operator.itemgetter(1))[1]
return (min(timeranges, key=operator.itemgetter(0))[0],
max(timeranges, key=operator.itemgetter(1))[1])
def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,
max_date: datetime, timeframe_mins: int) -> bool:
max_date: datetime, timeframe_min: int) -> bool:
"""
Validates preprocessed backtesting data for missing values and shows warnings about it that.
@@ -471,10 +466,10 @@ def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,
:param pair: pair used for log output.
:param min_date: start-date of the data
:param max_date: end-date of the data
:param timeframe_mins: ticker Timeframe in minutes
:param timeframe_min: ticker Timeframe in minutes
"""
# total difference in minutes / timeframe-minutes
expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_mins)
expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min)
found_missing = False
dflen = len(data)
if dflen < expected_frames:

15
freqtrade/edge/__init__.py
View File

@@ -80,7 +80,7 @@ class Edge:
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee()
self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
def calculate(self) -> bool:
pairs = self.config['exchange']['pair_whitelist']
@@ -94,12 +94,19 @@ class Edge:
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using local backtesting data (using whitelist in given config) ...')
if self._refresh_pairs:
history.refresh_data(
datadir=Path(self.config['datadir']),
pairs=pairs,
exchange=self.exchange,
timeframe=self.strategy.ticker_interval,
timerange=self._timerange,
)
data = history.load_data(
datadir=Path(self.config['datadir']),
pairs=pairs,
timeframe=self.strategy.ticker_interval,
refresh_pairs=self._refresh_pairs,
exchange=self.exchange,
timerange=self._timerange,
startup_candles=self.strategy.startup_candle_count,
)
@@ -113,7 +120,7 @@ class Edge:
preprocessed = self.strategy.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = history.get_timeframe(preprocessed)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days) ...',
min_date.isoformat(),

13
freqtrade/exchange/exchange.py
View File

@@ -18,7 +18,7 @@ from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP
from pandas import DataFrame
from freqtrade import (DependencyException, InvalidOrderException,
OperationalException, TemporaryError, constants)
OperationalException, TemporaryError)
from freqtrade.data.converter import parse_ticker_dataframe
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
from freqtrade.misc import deep_merge_dicts
@@ -379,15 +379,16 @@ class Exchange:
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{randint(0, 10**6)}'
_amount = self.symbol_amount_prec(pair, amount)
dry_order = {
"id": order_id,
'pair': pair,
'price': rate,
'amount': amount,
"cost": amount * rate,
'amount': _amount,
"cost": _amount * rate,
'type': ordertype,
'side': side,
'remaining': amount,
'remaining': _amount,
'datetime': arrow.utcnow().isoformat(),
'status': "closed" if ordertype == "market" else "open",
'fee': None,
@@ -478,7 +479,7 @@ class Exchange:
@retrier
def get_balance(self, currency: str) -> float:
if self._config['dry_run']:
return constants.DRY_RUN_WALLET
return self._config['dry_run_wallet']
# ccxt exception is already handled by get_balances
balances = self.get_balances()
@@ -920,7 +921,7 @@ class Exchange:
raise OperationalException(e) from e
@retrier
def get_fee(self, symbol='ETH/BTC', type='', side='', amount=1,
def get_fee(self, symbol, type='', side='', amount=1,
price=1, taker_or_maker='maker') -> float:
try:
# validate that markets are loaded before trying to get fee

15
freqtrade/freqtradebot.py
View File

@@ -62,7 +62,11 @@ class FreqtradeBot:
self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange
persistence.init(self.config.get('db_url', None),
clean_open_orders=self.config.get('dry_run', False))
self.wallets = Wallets(self.config, self.exchange)
self.dataprovider = DataProvider(self.config, self.exchange)
# Attach Dataprovider to Strategy baseclass
@@ -78,9 +82,6 @@ class FreqtradeBot:
self.active_pair_whitelist = self._refresh_whitelist()
persistence.init(self.config.get('db_url', None),
clean_open_orders=self.config.get('dry_run', False))
# Set initial bot state from config
initial_state = self.config.get('initial_state')
self.state = State[initial_state.upper()] if initial_state else State.STOPPED
@@ -231,8 +232,8 @@ class FreqtradeBot:
# Check if stake_amount is fulfilled
if available_amount < stake_amount:
raise DependencyException(
f"Available balance({available_amount} {self.config['stake_currency']}) is "
f"lower than stake amount({stake_amount} {self.config['stake_currency']})"
f"Available balance ({available_amount} {self.config['stake_currency']}) is "
f"lower than stake amount ({stake_amount} {self.config['stake_currency']})"
)
return stake_amount
@@ -554,6 +555,7 @@ class FreqtradeBot:
order['amount'] = new_amount
# Fee was applied, so set to 0
trade.fee_open = 0
trade.recalc_open_trade_price()
except DependencyException as exception:
logger.warning("Could not update trade amount: %s", exception)
@@ -849,6 +851,7 @@ class FreqtradeBot:
trade.amount = new_amount
# Fee was applied, so set to 0
trade.fee_open = 0
trade.recalc_open_trade_price()
except DependencyException as e:
logger.warning("Could not update trade amount: %s", e)
@@ -970,7 +973,7 @@ class FreqtradeBot:
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached ticker here - it was updated seconds ago.
current_rate = self.get_sell_rate(trade.pair, False)
profit_percent = trade.calc_profit_percent(profit_rate)
profit_percent = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_percent > 0 else "loss"
msg = {

76
freqtrade/optimize/backtesting.py
View File

@@ -65,7 +65,7 @@ class Backtesting:
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee()
self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
if self.config.get('runmode') != RunMode.HYPEROPT:
self.dataprovider = DataProvider(self.config, self.exchange)
@@ -87,7 +87,7 @@ class Backtesting:
raise OperationalException("Ticker-interval needs to be set in either configuration "
"or as cli argument `--ticker-interval 5m`")
self.timeframe = str(self.config.get('ticker_interval'))
self.timeframe_mins = timeframe_to_minutes(self.timeframe)
self.timeframe_min = timeframe_to_minutes(self.timeframe)
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
@@ -117,7 +117,7 @@ class Backtesting:
fail_without_data=True,
)
min_date, max_date = history.get_timeframe(data)
min_date, max_date = history.get_timerange(data)
logger.info(
'Loading data from %s up to %s (%s days)..',
@@ -261,6 +261,45 @@ class Backtesting:
ticker[pair] = [x for x in ticker_data.itertuples()]
return ticker
def _get_close_rate(self, sell_row, trade: Trade, sell, trade_dur) -> float:
"""
Get close rate for backtesting result
"""
# Special handling if high or low hit STOP_LOSS or ROI
if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
# Set close_rate to stoploss
return trade.stop_loss
elif sell.sell_type == (SellType.ROI):
roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur)
if roi is not None:
if roi == -1 and roi_entry % self.timeframe_min == 0:
# When forceselling with ROI=-1, the roi time will always be equal to trade_dur.
# If that entry is a multiple of the timeframe (so on candle open)
# - we'll use open instead of close
return sell_row.open
# - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
close_rate = - (trade.open_rate * roi + trade.open_rate *
(1 + trade.fee_open)) / (trade.fee_close - 1)
if (trade_dur > 0 and trade_dur == roi_entry
and roi_entry % self.timeframe_min == 0
and sell_row.open > close_rate):
# new ROI entry came into effect.
# use Open rate if open_rate > calculated sell rate
return sell_row.open
# Use the maximum between close_rate and low as we
# cannot sell outside of a candle.
# Applies when a new ROI setting comes in place and the whole candle is above that.
return max(close_rate, sell_row.low)
else:
# This should not be reached...
return sell_row.open
else:
return sell_row.open
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
partial_ticker: List, trade_count_lock: Dict,
@@ -287,29 +326,10 @@ class Backtesting:
sell_row.sell, low=sell_row.low, high=sell_row.high)
if sell.sell_flag:
trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60)
# Special handling if high or low hit STOP_LOSS or ROI
if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
# Set close_rate to stoploss
closerate = trade.stop_loss
elif sell.sell_type == (SellType.ROI):
roi = self.strategy.min_roi_reached_entry(trade_dur)
if roi is not None:
# - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
closerate = - (trade.open_rate * roi + trade.open_rate *
(1 + trade.fee_open)) / (trade.fee_close - 1)
# Use the maximum between closerate and low as we
# cannot sell outside of a candle.
# Applies when using {"xx": -1} as roi to force sells after xx minutes
closerate = max(closerate, sell_row.low)
else:
# This should not be reached...
closerate = sell_row.open
else:
closerate = sell_row.open
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=closerate),
profit_percent=trade.calc_profit_ratio(rate=closerate),
profit_abs=trade.calc_profit(rate=closerate),
open_time=buy_row.date,
close_time=sell_row.date,
@@ -325,7 +345,7 @@ class Backtesting:
# no sell condition found - trade stil open at end of backtest period
sell_row = partial_ticker[-1]
bt_res = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.open),
profit_percent=trade.calc_profit_ratio(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
open_time=buy_row.date,
close_time=sell_row.date,
@@ -378,7 +398,7 @@ class Backtesting:
lock_pair_until: Dict = {}
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = {}
tmp = start_date + timedelta(minutes=self.timeframe_mins)
tmp = start_date + timedelta(minutes=self.timeframe_min)
# Loop timerange and get candle for each pair at that point in time
while tmp < end_date:
@@ -430,7 +450,7 @@ class Backtesting:
lock_pair_until[pair] = end_date.datetime
# Move time one configured time_interval ahead.
tmp += timedelta(minutes=self.timeframe_mins)
tmp += timedelta(minutes=self.timeframe_min)
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def start(self) -> None:
@@ -461,7 +481,7 @@ class Backtesting:
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = history.trim_dataframe(df, timerange)
min_date, max_date = history.get_timeframe(preprocessed)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
'Backtesting with data from %s up to %s (%s days)..',

17
freqtrade/optimize/hyperopt.py
View File

@@ -6,6 +6,7 @@ This module contains the hyperopt logic
import locale
import logging
import random
import sys
import warnings
from collections import OrderedDict
@@ -22,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
from pandas import DataFrame
from freqtrade import OperationalException
from freqtrade.data.history import get_timeframe, trim_dataframe
from freqtrade.data.history import get_timerange, trim_dataframe
from freqtrade.misc import plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -368,7 +369,7 @@ class Hyperopt:
processed = load(self.tickerdata_pickle)
min_date, max_date = get_timeframe(processed)
min_date, max_date = get_timerange(processed)
backtesting_results = self.backtesting.backtest(
{
@@ -426,7 +427,7 @@ class Hyperopt:
f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
f"Avg duration {results_metrics['duration']:5.1f} mins."
f"Avg duration {results_metrics['duration']:5.1f} min."
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
@@ -436,7 +437,7 @@ class Hyperopt:
acq_optimizer="auto",
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.config.get('hyperopt_random_state', None),
random_state=self.random_state,
)
def fix_optimizer_models_list(self):
@@ -475,7 +476,13 @@ class Hyperopt:
logger.info(f"Loaded {len(trials)} previous evaluations from disk.")
return trials
def _set_random_state(self, random_state: Optional[int]) -> int:
return random_state or random.randint(1, 2**16 - 1)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
data, timerange = self.backtesting.load_bt_data()
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
@@ -483,7 +490,7 @@ class Hyperopt:
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = get_timeframe(data)
min_date, max_date = get_timerange(data)
logger.info(
'Hyperopting with data from %s up to %s (%s days)..',

6
freqtrade/optimize/hyperopt_interface.py
View File

@@ -106,7 +106,7 @@ class IHyperOpt(ABC):
roi_t_alpha = 1.0
roi_p_alpha = 1.0
timeframe_mins = timeframe_to_minutes(IHyperOpt.ticker_interval)
timeframe_min = timeframe_to_minutes(IHyperOpt.ticker_interval)
# We define here limits for the ROI space parameters automagically adapted to the
# timeframe used by the bot:
@@ -117,8 +117,8 @@ class IHyperOpt(ABC):
#
# The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space()
# method for the 5m ticker interval.
roi_t_scale = timeframe_mins / 5
roi_p_scale = math.log1p(timeframe_mins) / math.log1p(5)
roi_t_scale = timeframe_min / 5
roi_p_scale = math.log1p(timeframe_min) / math.log1p(5)
roi_limits = {
'roi_t1_min': int(10 * roi_t_scale * roi_t_alpha),
'roi_t1_max': int(120 * roi_t_scale * roi_t_alpha),

61
freqtrade/persistence.py
View File

@@ -86,7 +86,7 @@ def check_migrate(engine) -> None:
logger.debug(f'trying {table_back_name}')
# Check for latest column
if not has_column(cols, 'stop_loss_pct'):
if not has_column(cols, 'open_trade_price'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
@@ -104,6 +104,8 @@ def check_migrate(engine) -> None:
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
@@ -121,7 +123,7 @@ def check_migrate(engine) -> None:
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, strategy,
ticker_interval
ticker_interval, open_trade_price
)
select id, lower(exchange),
case
@@ -140,7 +142,8 @@ def check_migrate(engine) -> None:
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{strategy} strategy, {ticker_interval} ticker_interval
{strategy} strategy, {ticker_interval} ticker_interval,
{open_trade_price} open_trade_price
from {table_back_name}
""")
@@ -182,6 +185,8 @@ class Trade(_DECL_BASE):
fee_close = Column(Float, nullable=False, default=0.0)
open_rate = Column(Float)
open_rate_requested = Column(Float)
# open_trade_price - calcuated via _calc_open_trade_price
open_trade_price = Column(Float)
close_rate = Column(Float)
close_rate_requested = Column(Float)
close_profit = Column(Float)
@@ -210,6 +215,10 @@ class Trade(_DECL_BASE):
strategy = Column(String, nullable=True)
ticker_interval = Column(Integer, nullable=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.recalc_open_trade_price()
def __repr__(self):
open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed'
@@ -302,6 +311,7 @@ class Trade(_DECL_BASE):
# Update open rate and actual amount
self.open_rate = Decimal(order['price'])
self.amount = Decimal(order['amount'])
self.recalc_open_trade_price()
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
self.open_order_id = None
elif order_type in ('market', 'limit') and order['side'] == 'sell':
@@ -322,7 +332,7 @@ class Trade(_DECL_BASE):
and marks trade as closed
"""
self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_percent()
self.close_profit = self.calc_profit_ratio()
self.close_date = datetime.utcnow()
self.is_open = False
self.open_order_id = None
@@ -331,31 +341,36 @@ class Trade(_DECL_BASE):
self
)
def calc_open_trade_price(self, fee: Optional[float] = None) -> float:
def _calc_open_trade_price(self) -> float:
"""
Calculate the open_rate including fee.
:param fee: fee to use on the open rate (optional).
If rate is not set self.fee will be used
Calculate the open_rate including open_fee.
:return: Price in of the open trade incl. Fees
"""
buy_trade = (Decimal(self.amount) * Decimal(self.open_rate))
fees = buy_trade * Decimal(fee or self.fee_open)
buy_trade = Decimal(self.amount) * Decimal(self.open_rate)
fees = buy_trade * Decimal(self.fee_open)
return float(buy_trade + fees)
def recalc_open_trade_price(self) -> None:
"""
Recalculate open_trade_price.
Must be called whenever open_rate or fee_open is changed.
"""
self.open_trade_price = self._calc_open_trade_price()
def calc_close_trade_price(self, rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculate the close_rate including fee
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
If rate is not set self.fee will be used
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
If rate is not set self.close_rate will be used
:return: Price in BTC of the open trade
"""
if rate is None and not self.close_rate:
return 0.0
sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate))
sell_trade = Decimal(self.amount) * Decimal(rate or self.close_rate)
fees = sell_trade * Decimal(fee or self.fee_close)
return float(sell_trade - fees)
@@ -364,34 +379,32 @@ class Trade(_DECL_BASE):
"""
Calculate the absolute profit in stake currency between Close and Open trade
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
If rate is not set self.fee will be used
:param rate: close rate to compare with (optional).
If rate is not set self.close_rate will be used
If rate is not set self.close_rate will be used
:return: profit in stake currency as float
"""
open_trade_price = self.calc_open_trade_price()
close_trade_price = self.calc_close_trade_price(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
)
profit = close_trade_price - open_trade_price
profit = close_trade_price - self.open_trade_price
return float(f"{profit:.8f}")
def calc_profit_percent(self, rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
def calc_profit_ratio(self, rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculates the profit in percentage (including fee).
Calculates the profit as ratio (including fee).
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
If rate is not set self.close_rate will be used
:param fee: fee to use on the close rate (optional).
:return: profit in percentage as float
:return: profit ratio as float
"""
open_trade_price = self.calc_open_trade_price()
close_trade_price = self.calc_close_trade_price(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
)
profit_percent = (close_trade_price / open_trade_price) - 1
profit_percent = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_percent:.8f}")
@staticmethod

2
freqtrade/plot/plotting.py
View File

@@ -121,7 +121,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
)
# Create description for sell summarizing the trade
desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, "
f"{row['duration']}min",
f"{row['duration']} min",
axis=1)
trade_sells = go.Scatter(
x=trades["close_time"],

9
freqtrade/rpc/rpc.py
View File

@@ -123,7 +123,7 @@ class RPC:
current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
except DependencyException:
current_rate = NAN
current_profit = trade.calc_profit_percent(current_rate)
current_profit = trade.calc_profit_ratio(current_rate)
fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%'
if trade.close_profit else None)
trade_dict = trade.to_json()
@@ -151,7 +151,7 @@ class RPC:
current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
except DependencyException:
current_rate = NAN
trade_perc = (100 * trade.calc_profit_percent(current_rate))
trade_perc = (100 * trade.calc_profit_ratio(current_rate))
trade_profit = trade.calc_profit(current_rate)
profit_str = f'{trade_perc:.2f}%'
if self._fiat_converter:
@@ -240,7 +240,7 @@ class RPC:
durations.append((trade.close_date - trade.open_date).total_seconds())
if not trade.is_open:
profit_percent = trade.calc_profit_percent()
profit_percent = trade.calc_profit_ratio()
profit_closed_coin.append(trade.calc_profit())
profit_closed_perc.append(profit_percent)
else:
@@ -249,7 +249,7 @@ class RPC:
current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
except DependencyException:
current_rate = NAN
profit_percent = trade.calc_profit_percent(rate=current_rate)
profit_percent = trade.calc_profit_ratio(rate=current_rate)
profit_all_coin.append(
trade.calc_profit(rate=trade.close_rate or current_rate)
@@ -348,6 +348,7 @@ class RPC:
'total': total,
'symbol': symbol,
'value': value,
'note': 'Simulated balances' if self._freqtrade.config.get('dry_run', False) else ''
}
def _rpc_start(self) -> Dict[str, str]:

23
freqtrade/rpc/telegram.py
View File

@@ -331,7 +331,15 @@ class Telegram(RPC):
try:
result = self._rpc_balance(self._config['stake_currency'],
self._config.get('fiat_display_currency', ''))
output = ''
if self._config['dry_run']:
output += (
f"*Warning:*Simulated balances in Dry Mode.\n"
"This mode is still experimental!\n"
"Starting capital: "
f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n"
)
for currency in result['currencies']:
if currency['est_stake'] > 0.0001:
curr_output = "*{currency}:*\n" \
@@ -587,14 +595,25 @@ class Telegram(RPC):
:return: None
"""
val = self._rpc_show_config()
if val['trailing_stop']:
sl_info = (
f"*Initial Stoploss:* `{val['stoploss']}`\n"
f"*Trailing stop positive:* `{val['trailing_stop_positive']}`\n"
f"*Trailing stop offset:* `{val['trailing_stop_positive_offset']}`\n"
f"*Only trail above offset:* `{val['trailing_only_offset_is_reached']}`\n"
)
else:
sl_info = f"*Stoploss:* `{val['stoploss']}`\n"
self._send_msg(
f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n"
f"*Exchange:* `{val['exchange']}`\n"
f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n"
f"*Minimum ROI:* `{val['minimal_roi']}`\n"
f"*{'Trailing ' if val['trailing_stop'] else ''}Stoploss:* `{val['stoploss']}`\n"
f"{sl_info}"
f"*Ticker Interval:* `{val['ticker_interval']}`\n"
f"*Strategy:* `{val['strategy']}`'"
f"*Strategy:* `{val['strategy']}`"
)
def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:

14
freqtrade/strategy/interface.py
View File

@@ -296,7 +296,7 @@ class IStrategy(ABC):
"""
# Set current rate to low for backtesting sell
current_rate = low or rate
current_profit = trade.calc_profit_percent(current_rate)
current_profit = trade.calc_profit_ratio(current_rate)
trade.adjust_min_max_rates(high or current_rate)
@@ -311,7 +311,7 @@ class IStrategy(ABC):
# Set current rate to high for backtesting sell
current_rate = high or rate
current_profit = trade.calc_profit_percent(current_rate)
current_profit = trade.calc_profit_ratio(current_rate)
config_ask_strategy = self.config.get('ask_strategy', {})
if buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False):
@@ -360,7 +360,7 @@ class IStrategy(ABC):
sl_offset = self.trailing_stop_positive_offset
# Make sure current_profit is calculated using high for backtesting.
high_profit = current_profit if not high else trade.calc_profit_percent(high)
high_profit = current_profit if not high else trade.calc_profit_ratio(high)
# Don't update stoploss if trailing_only_offset_is_reached is true.
if not (self.trailing_only_offset_is_reached and high_profit < sl_offset):
@@ -394,7 +394,7 @@ class IStrategy(ABC):
return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
def min_roi_reached_entry(self, trade_dur: int) -> Optional[float]:
def min_roi_reached_entry(self, trade_dur: int) -> Tuple[Optional[int], Optional[float]]:
"""
Based on trade duration defines the ROI entry that may have been reached.
:param trade_dur: trade duration in minutes
@@ -403,9 +403,9 @@ class IStrategy(ABC):
# Get highest entry in ROI dict where key <= trade-duration
roi_list = list(filter(lambda x: x <= trade_dur, self.minimal_roi.keys()))
if not roi_list:
return None
return None, None
roi_entry = max(roi_list)
return self.minimal_roi[roi_entry]
return roi_entry, self.minimal_roi[roi_entry]
def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool:
"""
@@ -415,7 +415,7 @@ class IStrategy(ABC):
"""
# Check if time matches and current rate is above threshold
trade_dur = int((current_time.timestamp() - trade.open_date.timestamp()) // 60)
roi = self.min_roi_reached_entry(trade_dur)
_, roi = self.min_roi_reached_entry(trade_dur)
if roi is None:
return False
else:

2
freqtrade/utils.py
View File

@@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")

48
freqtrade/wallets.py
View File

@@ -4,7 +4,7 @@
import logging
from typing import Dict, NamedTuple, Any
from freqtrade.exchange import Exchange
from freqtrade import constants
from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
@@ -23,14 +23,12 @@ class Wallets:
self._config = config
self._exchange = exchange
self._wallets: Dict[str, Wallet] = {}
self.start_cap = config['dry_run_wallet']
self.update()
def get_free(self, currency) -> float:
if self._config['dry_run']:
return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
balance = self._wallets.get(currency)
if balance and balance.free:
return balance.free
@@ -39,9 +37,6 @@ class Wallets:
def get_used(self, currency) -> float:
if self._config['dry_run']:
return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
balance = self._wallets.get(currency)
if balance and balance.used:
return balance.used
@@ -50,16 +45,42 @@ class Wallets:
def get_total(self, currency) -> float:
if self._config['dry_run']:
return self._config.get('dry_run_wallet', constants.DRY_RUN_WALLET)
balance = self._wallets.get(currency)
if balance and balance.total:
return balance.total
else:
return 0
def update(self) -> None:
def _update_dry(self) -> None:
"""
Update from database in dry-run mode
- Apply apply profits of closed trades on top of stake amount
- Subtract currently tied up stake_amount in open trades
- update balances for currencies currently in trades
"""
closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all()
open_trades = Trade.get_trades(Trade.is_open.is_(True)).all()
tot_profit = sum([trade.calc_profit() for trade in closed_trades])
tot_in_trades = sum([trade.stake_amount for trade in open_trades])
current_stake = self.start_cap + tot_profit - tot_in_trades
self._wallets[self._config['stake_currency']] = Wallet(
self._config['stake_currency'],
current_stake,
0,
current_stake
)
for trade in open_trades:
curr = trade.pair.split('/')[0]
self._wallets[curr] = Wallet(
curr,
trade.amount,
0,
trade.amount
)
def _update_live(self) -> None:
balances = self._exchange.get_balances()
@@ -71,6 +92,11 @@ class Wallets:
balances[currency].get('total', None)
)
def update(self) -> None:
if self._config['dry_run']:
self._update_dry()
else:
self._update_live()
logger.info('Wallets synced.')
def get_all_balances(self) -> Dict[str, Any]: