adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into margin-db

Browse Source
This commit is contained in:
Sam Germain
2021-08-02 23:58:44 -06:00
parent ef429afb6f dfc17f2bd1
commit d88e2ae603
50 changed files with 827 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
freqtrade/configuration/configuration.py
View File

@@ -11,6 +11,7 @@ from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.exceptions import OperationalException
@@ -71,6 +72,11 @@ class Configuration:
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:

54
freqtrade/configuration/environment_vars.py Normal file
View File

@@ -0,0 +1,54 @@
import logging
import os
from typing import Any, Dict
from freqtrade.constants import ENV_VAR_PREFIX
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
def get_var_typed(val):
try:
return int(val)
except ValueError:
try:
return float(val)
except ValueError:
if val.lower() in ('t', 'true'):
return True
elif val.lower() in ('f', 'false'):
return False
# keep as string
return val
def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
"""
Environment variables must be prefixed with FREQTRADE.
FREQTRADE__{section}__{key}
:param env_dict: Dictionary to validate - usually os.environ
:param prefix: Prefix to consider (usually FREQTRADE__)
:return: Nested dict based on available and relevant variables.
"""
relevant_vars: Dict[str, Any] = {}
for env_var, val in sorted(env_dict.items()):
if env_var.startswith(prefix):
logger.info(f"Loading variable '{env_var}'")
key = env_var.replace(prefix, '')
for k in reversed(key.split('__')):
val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
relevant_vars = deep_merge_dicts(val, relevant_vars)
return relevant_vars
def enironment_vars_to_dict() -> Dict[str, Any]:
"""
Read environment variables and return a nested dict for relevant variables
Relevant variables must follow the FREQTRADE__{section}__{key} pattern
:return: Nested dict based on available and relevant variables.
"""
return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

1
freqtrade/constants.py
View File

@@ -47,6 +47,7 @@ USERPATH_STRATEGIES = 'strategies'
USERPATH_NOTEBOOKS = 'notebooks'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
ENV_VAR_PREFIX = 'FREQTRADE__'
# Define decimals per coin for outputs

2
freqtrade/data/btanalysis.py
View File

@@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'sell_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:

25
freqtrade/data/dataprovider.py
View File

@@ -10,11 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.data.history import load_pair_history
from freqtrade.enums import RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange
from freqtrade.exchange import Exchange, timeframe_to_seconds
logger = logging.getLogger(__name__)
@@ -31,6 +32,7 @@ class DataProvider:
self._pairlists = pairlists
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
def _set_dataframe_max_index(self, limit_index: int):
"""
@@ -62,11 +64,22 @@ class DataProvider:
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
"""
return load_pair_history(pair=pair,
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
data_format=self._config.get('dataformat_ohlcv', 'json')
)
saved_pair = (pair, str(timeframe))
if saved_pair not in self.__cached_pairs_backtesting:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
# Move informative start time respecting startup_candle_count
timerange.subtract_start(
timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
)
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
pair=pair,
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
timerange=timerange,
data_format=self._config.get('dataformat_ohlcv', 'json')
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
"""

2
freqtrade/data/history/jsondatahandler.py
View File

@@ -62,7 +62,7 @@ class JsonDataHandler(IDataHandler):
filename = self._pair_data_filename(self._datadir, pair, timeframe)
_data = data.copy()
# Convert date to int
_data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
_data['date'] = _data['date'].view(np.int64) // 1000 // 1000
# Reset index, select only appropriate columns and save as json
_data.reset_index(drop=True).loc[:, self._columns].to_json(

2
freqtrade/enums/__init__.py
View File

@@ -4,5 +4,5 @@ from freqtrade.enums.interestmode import InterestMode
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
from freqtrade.enums.selltype import SellType
from freqtrade.enums.signaltype import SignalType
from freqtrade.enums.signaltype import SignalTagType, SignalType
from freqtrade.enums.state import State

7
freqtrade/enums/signaltype.py
View File

@@ -7,3 +7,10 @@ class SignalType(Enum):
"""
BUY = "buy"
SELL = "sell"
class SignalTagType(Enum):
"""
Enum for signal columns
"""
BUY_TAG = "buy_tag"

39
freqtrade/exchange/exchange.py
View File

@@ -387,7 +387,7 @@ class Exchange:
# its contents depend on the exchange.
# It can also be a string or similar ... so we need to verify that first.
elif (isinstance(self.markets[pair].get('info', None), dict)
and self.markets[pair].get('info', {}).get('IsRestricted', False)):
and self.markets[pair].get('info', {}).get('prohibitedIn', False)):
# Warn users about restricted pairs in whitelist.
# We cannot determine reliably if Users are affected.
logger.warning(f"Pair {pair} is restricted for some users on this exchange."
@@ -689,7 +689,16 @@ class Exchange:
# Order handling
def create_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict:
rate: float, time_in_force: str = 'gtc') -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
params.update({'timeInForce': time_in_force})
try:
# Set the precision for amount and price(rate) as accepted by the exchange
amount = self.amount_to_precision(pair, amount)
@@ -720,32 +729,6 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def buy(self, pair: str, ordertype: str, amount: float,
rate: float, time_in_force: str) -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
params.update({'timeInForce': time_in_force})
return self.create_order(pair, ordertype, 'buy', amount, rate, params)
def sell(self, pair: str, ordertype: str, amount: float,
rate: float, time_in_force: str = 'gtc') -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
params.update({'timeInForce': time_in_force})
return self.create_order(pair, ordertype, 'sell', amount, rate, params)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)

40
freqtrade/freqtradebot.py
View File

@@ -420,20 +420,24 @@ class FreqtradeBot(LoggingMixin):
return False
# running get_signal on historical data fetched
(buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
(buy, sell, buy_tag) = self.strategy.get_signal(
pair,
self.strategy.timeframe,
analyzed_df
)
if buy and not sell:
stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
if ((bid_check_dom.get('enabled', False)) and
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
if self._check_depth_of_market_buy(pair, bid_check_dom):
return self.execute_buy(pair, stake_amount)
return self.execute_buy(pair, stake_amount, buy_tag=buy_tag)
else:
return False
return self.execute_buy(pair, stake_amount)
return self.execute_buy(pair, stake_amount, buy_tag=buy_tag)
else:
return False
@@ -462,7 +466,7 @@ class FreqtradeBot(LoggingMixin):
return False
def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None,
forcebuy: bool = False) -> bool:
forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
"""
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
@@ -510,9 +514,9 @@ class FreqtradeBot(LoggingMixin):
logger.info(f"User requested abortion of buying {pair}")
return False
amount = self.exchange.amount_to_precision(pair, amount)
order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy",
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
order_id = order['id']
order_status = order.get('status', None)
@@ -565,6 +569,7 @@ class FreqtradeBot(LoggingMixin):
exchange=self.exchange.id,
open_order_id=order_id,
strategy=self.strategy.get_strategy_name(),
buy_tag=buy_tag,
timeframe=timeframe_to_minutes(self.config['timeframe'])
)
trade.orders.append(order_obj)
@@ -590,6 +595,7 @@ class FreqtradeBot(LoggingMixin):
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@@ -614,6 +620,7 @@ class FreqtradeBot(LoggingMixin):
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_CANCEL,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@@ -634,6 +641,7 @@ class FreqtradeBot(LoggingMixin):
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_FILL,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'open_rate': trade.open_rate,
@@ -692,7 +700,11 @@ class FreqtradeBot(LoggingMixin):
analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
self.strategy.timeframe)
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
(buy, sell, _) = self.strategy.get_signal(
trade.pair,
self.strategy.timeframe,
analyzed_df
)
logger.debug('checking sell')
sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell")
@@ -1094,11 +1106,11 @@ class FreqtradeBot(LoggingMixin):
try:
# Execute sell and update trade record
order = self.exchange.sell(pair=trade.pair,
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
order = self.exchange.create_order(pair=trade.pair,
ordertype=order_type, side="sell",
amount=amount, rate=limit,
time_in_force=time_in_force
)
except InsufficientFundsError as e:
logger.warning(f"Unable to place order {e}.")
# Try to figure out what went wrong

53
freqtrade/optimize/backtesting.py
View File

@@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.converter import trim_dataframe, trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import BacktestState, SellType
from freqtrade.exceptions import DependencyException, OperationalException
@@ -43,6 +43,7 @@ CLOSE_IDX = 3
SELL_IDX = 4
LOW_IDX = 5
HIGH_IDX = 6
BUY_TAG_IDX = 7
class Backtesting:
@@ -116,8 +117,13 @@ class Backtesting:
self.wallets = Wallets(self.config, self.exchange, log=False)
self.timerange = TimeRange.parse_timerange(
None if self.config.get('timerange') is None else str(self.config.get('timerange')))
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
# Add maximum startup candle count to configuration for informative pairs support
self.config['startup_candle_count'] = self.required_startup
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.progress = BTProgress()
@@ -154,14 +160,11 @@ class Backtesting:
"""
self.progress.init_step(BacktestState.DATALOAD, 1)
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe,
timerange=timerange,
timerange=self.timerange,
startup_candles=self.required_startup,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
@@ -174,11 +177,11 @@ class Backtesting:
f'({(max_date - min_date).days} days).')
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
self.progress.set_new_value(1)
return data, timerange
return data, self.timerange
def prepare_backtest(self, enable_protections):
"""
@@ -217,23 +220,34 @@ class Backtesting:
for pair, pair_data in processed.items():
self.check_abort()
self.progress.increment()
has_buy_tag = 'buy_tag' in pair_data
headers = headers + ['buy_tag'] if has_buy_tag else headers
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
if has_buy_tag:
pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy()
# Trim startup period from analyzed dataframe
df_analyzed = trim_dataframe(df_analyzed, self.timerange,
startup_candles=self.required_startup)
# To avoid using data from future, we use buy/sell signals shifted
# from the previous candle
df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
if has_buy_tag:
df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
# Update dataprovider cache
self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed.values.tolist()
data[pair] = df_analyzed[headers].values.tolist()
return data
def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@@ -262,7 +276,7 @@ class Backtesting:
# Worst case: price reaches stop_positive_offset and dives down.
stop_rate = (sell_row[OPEN_IDX] *
(1 + abs(self.strategy.trailing_stop_positive_offset) -
abs(self.strategy.trailing_stop_positive)))
abs(self.strategy.trailing_stop_positive)))
else:
# Worst case: price ticks tiny bit above open and dives down.
stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct))
@@ -358,6 +372,7 @@ class Backtesting:
if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
# Enter trade
has_buy_tag = len(row) >= BUY_TAG_IDX + 1
trade = LocalTrade(
pair=pair,
open_rate=row[OPEN_IDX],
@@ -367,6 +382,7 @@ class Backtesting:
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
buy_tag=row[BUY_TAG_IDX] if has_buy_tag else None,
exchange='backtesting',
)
return trade
@@ -423,10 +439,6 @@ class Backtesting:
trades: List[LocalTrade] = []
self.prepare_backtest(enable_protections)
# Update dataprovider cache
for pair, dataframe in processed.items():
self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
@@ -537,14 +549,15 @@ class Backtesting:
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
if not preprocessed:
if not preprocessed_tmp:
raise OperationalException(
"No data left after adjusting for startup candles.")
min_date, max_date = history.get_timerange(preprocessed)
# Use preprocessed_tmp for date generation (the trimmed dataframe).
# Backtesting will re-trim the dataframes after buy/sell signal generation.
min_date, max_date = history.get_timerange(preprocessed_tmp)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days).')

40
freqtrade/optimize/hyperopt.py
View File

@@ -102,16 +102,18 @@ class Hyperopt:
self.num_epochs_saved = 0
self.current_best_epoch: Optional[Dict[str, Any]] = None
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
if not self.auto_hyperopt:
# Populate "fallback" functions here
# (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
@@ -264,17 +266,14 @@ class Hyperopt:
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
"""
Used Optimize function. Called once per epoch to optimize whatever is configured.
Used Optimize function.
Called once per epoch to optimize whatever is configured.
Keep this function as optimized as possible!
"""
backtest_start_time = datetime.now(timezone.utc)
params_dict = self._get_params_dict(self.dimensions, raw_params)
# Apply parameters
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if HyperoptTools.has_space(self.config, 'buy'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
@@ -283,6 +282,10 @@ class Hyperopt:
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if HyperoptTools.has_space(self.config, 'stoploss'):
self.backtesting.strategy.stoploss = params_dict['stoploss']
@@ -378,16 +381,15 @@ class Hyperopt:
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
# Trim startup period from analyzed dataframe to get correct dates for output.
processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
self.min_date, self.max_date = get_timerange(processed)
logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(self.max_date - self.min_date).days} days)..')
dump(processed, self.data_pickle_file)
# Store non-trimmed data - will be trimmed after signal generation.
dump(preprocessed, self.data_pickle_file)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))

25
freqtrade/persistence/migrations.py
View File

@@ -54,6 +54,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
# sqlite does not support literals for booleans
is_short = get_column_def(cols, 'is_short', '0')
interest_mode = get_column_def(cols, 'interest_mode', 'null')
buy_tag = get_column_def(cols, 'buy_tag', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
@@ -72,7 +73,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
# Schema migration necessary
with engine.begin() as connection:
connection.execute(text(f"alter table trades rename to {table_back_name}"))
# drop indexes on backup table
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
# let SQLAlchemy create the schema as required
@@ -83,23 +85,16 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
connection.execute(text(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
fee_close, fee_close_cost, fee_close_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
timeframe, open_trade_value, close_profit_abs,
leverage, interest_rate, isolated_liq, is_short, interest_mode
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
select id, lower(exchange), pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -112,7 +107,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs,
{leverage} leverage, {interest_rate} interest_rate,
{isolated_liq} isolated_liq, {is_short} is_short,
@@ -143,7 +138,9 @@ def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, col
with engine.begin() as connection:
connection.execute(text(f"alter table orders rename to {table_back_name}"))
# drop indexes on backup table
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
@@ -174,7 +171,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
table_back_name = get_backup_name(tabs, 'trades_bak')
# Check for latest column
if not has_column(cols, 'open_trade_value'):
if not has_column(cols, 'buy_tag'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!

5
freqtrade/persistence/models.py
View File

@@ -261,6 +261,7 @@ class LocalTrade():
sell_reason: str = ''
sell_order_status: str = ''
strategy: str = ''
buy_tag: Optional[str] = None
timeframe: Optional[int] = None
# Margin trading properties
@@ -378,6 +379,7 @@ class LocalTrade():
'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
'stake_amount': round(self.stake_amount, 8),
'strategy': self.strategy,
'buy_tag': self.buy_tag,
'timeframe': self.timeframe,
'fee_open': self.fee_open,
@@ -817,7 +819,7 @@ class LocalTrade():
# skip case if trailing-stop changed the stoploss already.
if (trade.stop_loss == trade.initial_stop_loss
and trade.initial_stop_loss_pct != desired_stoploss):
and trade.initial_stop_loss_pct != desired_stoploss):
# Stoploss value got changed
logger.info(f"Stoploss for {trade} needs adjustment...")
@@ -884,6 +886,7 @@ class Trade(_DECL_BASE, LocalTrade):
sell_reason = Column(String(100), nullable=True) # TODO-mg: Change to close_reason
sell_order_status = Column(String(100), nullable=True) # TODO-mg: Change to close_order_status
strategy = Column(String(100), nullable=True)
buy_tag = Column(String(100), nullable=True)
timeframe = Column(Integer, nullable=True)
# Margin trading properties

17
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@@ -26,6 +26,7 @@ class RangeStabilityFilter(IPairList):
self._days = pairlistconfig.get('lookback_days', 10)
self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -50,8 +51,12 @@ class RangeStabilityFilter(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
max_rate_desc = ""
if self._max_rate_of_change:
max_rate_desc = (f" and above {self._max_rate_of_change}")
return (f"{self.name} - Filtering pairs with rate of change below "
f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
f"{self._min_rate_of_change}{max_rate_desc} over the "
f"last {plural(self._days, 'day')}.")
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
@@ -104,6 +109,16 @@ class RangeStabilityFilter(IPairList):
f"which is below the threshold of {self._min_rate_of_change}.",
logger.info)
result = False
if self._max_rate_of_change:
if pct_change <= self._max_rate_of_change:
result = True
else:
self.log_once(
f"Removed {pair} from whitelist, because rate of change "
f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
f"which is above the threshold of {self._max_rate_of_change}.",
logger.info)
result = False
self._pair_cache[pair] = result
return result

4
freqtrade/rpc/api_server/api_backtest.py
View File

@@ -47,15 +47,15 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
or lastconfig.get('timerange') != btconfig['timerange']
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
# Only reload data if timeframe or timerange changed.
# Only reload data if timeframe changed.
if (
not ApiServer._bt_data
or not ApiServer._bt_timerange
or lastconfig.get('timerange') != btconfig['timerange']
or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
or lastconfig.get('protections') != btconfig.get('protections', [])

1
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -151,6 +151,7 @@ class TradeSchema(BaseModel):
amount_requested: float
stake_amount: float
strategy: str
buy_tag: Optional[str]
timeframe: int
fee_open: Optional[float]
fee_open_cost: Optional[float]

27
freqtrade/rpc/telegram.py
View File

@@ -208,15 +208,25 @@ class Telegram(RPCHandler):
else:
msg['stake_amount_fiat'] = 0
message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
f" (#{msg['trade_id']})\n"
f"*Amount:* `{msg['amount']:.8f}`\n"
f"*Open Rate:* `{msg['limit']:.8f}`\n"
f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
content = []
content.append(
f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
f" (#{msg['trade_id']})\n"
)
if msg.get('buy_tag', None):
content.append(f"*Buy Tag:* `{msg['buy_tag']}`\n")
content.append(f"*Amount:* `{msg['amount']:.8f}`\n")
content.append(f"*Open Rate:* `{msg['limit']:.8f}`\n")
content.append(f"*Current Rate:* `{msg['current_rate']:.8f}`\n")
content.append(
f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}"
)
if msg.get('fiat_currency', None):
message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
content.append(
f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
)
message = ''.join(content)
message += ")`"
return message
@@ -354,6 +364,7 @@ class Telegram(RPCHandler):
"*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
"*Current Pair:* {pair}",
"*Amount:* `{amount} ({stake_amount} {base_currency})`",
"*Buy Tag:* `{buy_tag}`" if r['buy_tag'] else "",
"*Open Rate:* `{open_rate:.8f}`",
"*Close Rate:* `{close_rate}`" if r['close_rate'] else "",
"*Current Rate:* `{current_rate:.8f}`",

29
freqtrade/strategy/interface.py
View File

@@ -13,7 +13,7 @@ from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import SellType, SignalType
from freqtrade.enums import SellType, SignalTagType, SignalType
from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date
@@ -422,6 +422,7 @@ class IStrategy(ABC, HyperStrategyMixin):
logger.debug("Skipping TA Analysis for already analyzed candle")
dataframe['buy'] = 0
dataframe['sell'] = 0
dataframe['buy_tag'] = None
# Other Defs in strategy that want to be called every loop here
# twitter_sell = self.watch_twitter_feed(dataframe, metadata)
@@ -482,8 +483,6 @@ class IStrategy(ABC, HyperStrategyMixin):
message = "No dataframe returned (return statement missing?)."
elif 'buy' not in dataframe:
message = "Buy column not set."
elif 'sell' not in dataframe:
message = "Sell column not set."
elif df_len != len(dataframe):
message = message_template.format("length")
elif df_close != dataframe["close"].iloc[-1]:
@@ -496,7 +495,12 @@ class IStrategy(ABC, HyperStrategyMixin):
else:
raise StrategyError(message)
def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
def get_signal(
self,
pair: str,
timeframe: str,
dataframe: DataFrame
) -> Tuple[bool, bool, Optional[str]]:
"""
Calculates current signal based based on the buy / sell columns of the dataframe.
Used by Bot to get the signal to buy or sell
@@ -507,7 +511,7 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
return False, False
return False, False, None
latest_date = dataframe['date'].max()
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
@@ -522,9 +526,16 @@ class IStrategy(ABC, HyperStrategyMixin):
'Outdated history for pair %s. Last tick is %s minutes old',
pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
)
return False, False
return False, False, None
buy = latest[SignalType.BUY.value] == 1
sell = False
if SignalType.SELL.value in latest:
sell = latest[SignalType.SELL.value] == 1
buy_tag = latest.get(SignalTagType.BUY_TAG.value, None)
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'], pair, str(buy), str(sell))
timeframe_seconds = timeframe_to_seconds(timeframe)
@@ -532,8 +543,8 @@ class IStrategy(ABC, HyperStrategyMixin):
current_time=datetime.now(timezone.utc),
timeframe_seconds=timeframe_seconds,
buy=buy):
return False, sell
return buy, sell
return False, sell, buy_tag
return buy, sell, buy_tag
def ignore_expired_candle(self, latest_date: datetime, current_time: datetime,
timeframe_seconds: int, buy: bool):