adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into pr/rextea/4606

Browse Source
This commit is contained in:
Matthias
2021-10-17 16:29:19 +02:00
parent 76a02ff70a 9bb2dd1851
commit 47bba331c1
287 changed files with 17159 additions and 9644 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
freqtrade/configuration/PeriodicCache.py Normal file
View File

@@ -0,0 +1,19 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
class PeriodicCache(TTLCache):
"""
Special cache that expires at "straight" times
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
"""
def __init__(self, maxsize, ttl, getsizeof=None):
def local_timer():
ts = datetime.now(timezone.utc).timestamp()
offset = (ts % ttl)
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

3
freqtrade/configuration/__init__.py
View File

@@ -1,7 +1,8 @@
# flake8: noqa: F401
from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange

37
freqtrade/configuration/check_exchange.py
View File

@@ -1,28 +1,15 @@
import logging
from typing import Any, Dict
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported)
from freqtrade.state import RunMode
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
is_exchange_officially_supported, validate_exchange)
logger = logging.getLogger(__name__)
def remove_credentials(config: Dict[str, Any]) -> None:
"""
Removes exchange keys from the configuration and specifies dry-run
Used for backtesting / hyperopt / edge and utils.
Modifies the input dict!
"""
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
config['exchange']['password'] = ''
config['exchange']['uid'] = ''
config['dry_run'] = True
def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
"""
Check if the exchange name in the config file is supported by Freqtrade
@@ -51,15 +38,19 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
if not is_exchange_known_ccxt(exchange):
raise OperationalException(
f'Exchange "{exchange}" is not known to the ccxt library '
f'and therefore not available for the bot.\n'
f'The following exchanges are available for Freqtrade: '
f'{", ".join(available_exchanges())}'
f'Exchange "{exchange}" is not known to the ccxt library '
f'and therefore not available for the bot.\n'
f'The following exchanges are available for Freqtrade: '
f'{", ".join(available_exchanges())}'
)
if check_for_bad and is_exchange_bad(exchange):
raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. '
f'Reason: {get_exchange_bad_reason(exchange)}')
valid, reason = validate_exchange(exchange)
if not valid:
if check_for_bad:
raise OperationalException(f'Exchange "{exchange}" will not work with Freqtrade. '
f'Reason: {reason}')
else:
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
if is_exchange_officially_supported(exchange):
logger.info(f'Exchange "{exchange}" is officially supported '

8
freqtrade/configuration/config_setup.py
View File

@@ -1,9 +1,8 @@
import logging
from typing import Any, Dict
from freqtrade.state import RunMode
from freqtrade.enums import RunMode
from .check_exchange import remove_credentials
from .config_validation import validate_config_consistency
from .configuration import Configuration
@@ -15,13 +14,14 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
"""
Prepare the configuration for utils subcommands
:param args: Cli args from Arguments()
:param method: Bot running mode
:return: Configuration
"""
configuration = Configuration(args, method)
config = configuration.get_config()
# Ensure we do not use Exchange credentials
remove_credentials(config)
# Ensure these modes are using Dry-run
config['dry_run'] = True
validate_config_consistency(config)
return config

48
freqtrade/configuration/config_validation.py
View File

@@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators
from jsonschema.exceptions import ValidationError, best_match
from freqtrade import constants
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -74,10 +74,12 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
# validating trailing stoploss
_validate_trailing_stoploss(conf)
_validate_price_config(conf)
_validate_edge(conf)
_validate_whitelist(conf)
_validate_protections(conf)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
# validate configuration before returning
logger.info('Validating configuration ...')
@@ -95,12 +97,25 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.")
def _validate_price_config(conf: Dict[str, Any]) -> None:
"""
When using market orders, price sides must be using the "other" side of the price
"""
if (conf.get('order_types', {}).get('buy') == 'market'
and conf.get('bid_strategy', {}).get('price_side') != 'ask'):
raise OperationalException('Market buy orders require bid_strategy.price_side = "ask".')
if (conf.get('order_types', {}).get('sell') == 'market'
and conf.get('ask_strategy', {}).get('price_side') != 'bid'):
raise OperationalException('Market sell orders require ask_strategy.price_side = "bid".')
def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
if conf.get('stoploss') == 0.0:
raise OperationalException(
'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
)
)
# Skip if trailing stoploss is not activated
if not conf.get('trailing_stop', False):
return
@@ -135,12 +150,7 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
if not conf.get('edge', {}).get('enabled'):
return
if conf.get('pairlist', {}).get('method') == 'VolumePairList':
raise OperationalException(
"Edge and VolumePairList are incompatible, "
"Edge will override whatever pairs VolumePairlist selects."
)
if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
if not conf.get('use_sell_signal', True):
raise OperationalException(
"Edge requires `use_sell_signal` to be True, otherwise no sells will happen."
)
@@ -170,10 +180,30 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
raise OperationalException(
"Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
f"Please fix the protection {prot.get('method')}"
)
)
if ('lookback_period' in prot and 'lookback_period_candles' in prot):
raise OperationalException(
"Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
f"Please fix the protection {prot.get('method')}"
)
def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
ask_strategy = conf.get('ask_strategy', {})
ob_min = ask_strategy.get('order_book_min')
ob_max = ask_strategy.get('order_book_max')
if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
if ob_min != ob_max:
raise OperationalException(
"Using order_book_max != order_book_min in ask_strategy is no longer supported."
"Please pick one value and use `order_book_top` in the future."
)
else:
# Move value to order_book_top
ask_strategy['order_book_top'] = ob_min
logger.warning(
"DEPRECATED: "
"Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
"for your `ask_strategy` configuration."
)

46
freqtrade/configuration/configuration.py
View File

@@ -11,11 +11,12 @@ from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.load_config import load_config_file
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
logger = logging.getLogger(__name__)
@@ -72,11 +73,14 @@ class Configuration:
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}
# TODO: This can be deleted along with removal of deprecated
# experimental settings
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
@@ -108,6 +112,8 @@ class Configuration:
self._process_plot_options(config)
self._process_data_options(config)
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
@@ -144,7 +150,7 @@ class Configuration:
config['db_url'] = constants.DEFAULT_DB_PROD_URL
logger.info('Dry run is disabled')
logger.info(f'Using DB: "{config["db_url"]}"')
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
def _process_common_options(self, config: Dict[str, Any]) -> None:
@@ -236,6 +242,9 @@ class Configuration:
except ValueError:
pass
self._args_to_config(config, argname='timeframe_detail',
logstring='Parameter --timeframe-detail detected, '
'using {} for intra-candle backtesting ...')
self._args_to_config(config, argname='stake_amount',
logstring='Parameter --stake-amount detected, '
'overriding stake_amount to: {} ...')
@@ -263,6 +272,9 @@ class Configuration:
self._args_to_config(config, argname='show_days',
logstring='Parameter --show-days detected ...')
self._args_to_config(config, argname='disableparamexport',
logstring='Parameter --disableparamexport detected: {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
txt_range = eval(self.args["stoploss_range"])
@@ -361,6 +373,9 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_show_no_header',
logstring='Parameter --no-header detected: {}')
self._args_to_config(config, argname="hyperopt_ignore_missing_space",
logstring="Paramter --ignore-missing-space detected: {}")
def _process_plot_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='pairs',
@@ -378,6 +393,9 @@ class Configuration:
self._args_to_config(config, argname='plot_limit',
logstring='Limiting plot to: {}')
self._args_to_config(config, argname='plot_auto_open',
logstring='Parameter --auto-open detected.')
self._args_to_config(config, argname='trade_source',
logstring='Using trades from: {}')
@@ -402,6 +420,11 @@ class Configuration:
self._args_to_config(config, argname='dataformat_trades',
logstring='Using "{}" to store trades data.')
def _process_data_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='new_pairs_days',
logstring='Detected --new-pairs-days: {}')
def _process_runmode(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='dry_run',
@@ -448,18 +471,18 @@ class Configuration:
"""
if "pairs" in config:
config['exchange']['pair_whitelist'] = config['pairs']
return
if "pairs_file" in self.args and self.args["pairs_file"]:
pairs_file = Path(self.args["pairs_file"])
logger.info(f'Reading pairs file "{pairs_file}".')
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
# or if pairs file is specified explicitly
if not pairs_file.exists():
raise OperationalException(f'No pairs file found with path "{pairs_file}".')
with pairs_file.open('r') as f:
config['pairs'] = json_load(f)
config['pairs'].sort()
config['pairs'] = load_file(pairs_file)
config['pairs'].sort()
return
if 'config' in self.args and self.args['config']:
@@ -469,7 +492,6 @@ class Configuration:
# Fall back to /dl_path/pairs.json
pairs_file = config['datadir'] / 'pairs.json'
if pairs_file.exists():
with pairs_file.open('r') as f:
config['pairs'] = json_load(f)
config['pairs'] = load_file(pairs_file)
if 'pairs' in config:
config['pairs'].sort()

68
freqtrade/configuration/deprecated_settings.py
View File

@@ -3,7 +3,7 @@ Functions to handle deprecated settings
"""
import logging
from typing import Any, Dict
from typing import Any, Dict, Optional
from freqtrade.exceptions import OperationalException
@@ -12,23 +12,24 @@ logger = logging.getLogger(__name__)
def check_conflicting_settings(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section1_config = config.get(section1, {})
section2_config = config.get(section2, {})
if name1 in section1_config and name2 in section2_config:
section_old: str, name_old: str,
section_new: Optional[str], name_new: str) -> None:
section_new_config = config.get(section_new, {}) if section_new else config
section_old_config = config.get(section_old, {})
if name_new in section_new_config and name_old in section_old_config:
new_name = f"{section_new}.{name_new}" if section_new else f"{name_new}"
raise OperationalException(
f"Conflicting settings `{section1}.{name1}` and `{section2}.{name2}` "
f"Conflicting settings `{new_name}` and `{section_old}.{name_old}` "
"(DEPRECATED) detected in the configuration file. "
"This deprecated setting will be removed in the next versions of Freqtrade. "
f"Please delete it from your configuration and use the `{section1}.{name1}` "
f"Please delete it from your configuration and use the `{new_name}` "
"setting instead."
)
def process_removed_setting(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section2: Optional[str], name2: str) -> None:
"""
:param section1: Removed section
:param name1: Removed setting name
@@ -37,27 +38,32 @@ def process_removed_setting(config: Dict[str, Any],
"""
section1_config = config.get(section1, {})
if name1 in section1_config:
section_2 = f"{section2}.{name2}" if section2 else f"{name2}"
raise OperationalException(
f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. "
f"Please delete it from your configuration and use the `{section2}.{name2}` "
f"Setting `{section1}.{name1}` has been moved to `{section_2}. "
f"Please delete it from your configuration and use the `{section_2}` "
"setting instead."
)
def process_deprecated_setting(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section2_config = config.get(section2, {})
section_old: str, name_old: str,
section_new: Optional[str], name_new: str
) -> None:
check_conflicting_settings(config, section_old, name_old, section_new, name_new)
section_old_config = config.get(section_old, {})
if name2 in section2_config:
if name_old in section_old_config:
section_2 = f"{section_new}.{name_new}" if section_new else f"{name_new}"
logger.warning(
"DEPRECATED: "
f"The `{section2}.{name2}` setting is deprecated and "
f"The `{section_old}.{name_old}` setting is deprecated and "
"will be removed in the next versions of Freqtrade. "
f"Please use the `{section1}.{name1}` setting in your configuration instead."
f"Please use the `{section_2}` setting in your configuration instead."
)
section1_config = config.get(section1, {})
section1_config[name1] = section2_config[name2]
section_new_config = config.get(section_new, {}) if section_new else config
section_new_config[name_new] = section_old_config[name_old]
def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
@@ -65,15 +71,24 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
# Kept for future deprecated / moved settings
# check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
# 'experimental', 'use_sell_signal')
# process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
# 'experimental', 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
None, 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
None, 'sell_profit_only')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_offset',
None, 'sell_profit_offset')
process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_buy_signal')
process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
None, 'ignore_buying_expired_candle_after')
# Legacy way - having them in experimental ...
process_removed_setting(config, 'experimental', 'use_sell_signal',
'ask_strategy', 'use_sell_signal')
None, 'use_sell_signal')
process_removed_setting(config, 'experimental', 'sell_profit_only',
'ask_strategy', 'sell_profit_only')
None, 'sell_profit_only')
process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
'ask_strategy', 'ignore_roi_if_buy_signal')
None, 'ignore_roi_if_buy_signal')
if (config.get('edge', {}).get('enabled', False)
and 'capital_available_percentage' in config.get('edge', {})):
@@ -93,5 +108,8 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
raise OperationalException(
"Both 'timeframe' and 'ticker_interval' detected."
"Please remove 'ticker_interval' from your configuration to continue operating."
)
)
config['timeframe'] = config['ticker_interval']
if 'protections' in config:
logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

19
freqtrade/configuration/directory_operations.py
View File

@@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat
return folder
def chown_user_directory(directory: Path) -> None:
"""
Use Sudo to change permissions of the home-directory if necessary
Only applies when running in docker!
"""
import os
if os.environ.get('FT_APP_ENV') == 'docker':
try:
import subprocess
subprocess.check_output(
['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
except Exception:
logger.warning(f"Could not chown {directory}")
def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
"""
Create userdata directory structure.
@@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
"notebooks", "plot", "strategies", ]
folder = Path(directory)
chown_user_directory(folder)
if not folder.is_dir():
if create_dir:
folder.mkdir(parents=True)
@@ -72,6 +88,5 @@ def copy_sample_files(directory: Path, overwrite: bool = False) -> None:
if not overwrite:
logger.warning(f"File `{targetfile}` exists already, not deploying sample file.")
continue
else:
logger.warning(f"File `{targetfile}` exists already, overwriting.")
logger.warning(f"File `{targetfile}` exists already, overwriting.")
shutil.copy(str(sourcedir / source), str(targetfile))

54
freqtrade/configuration/environment_vars.py Normal file
View File

@@ -0,0 +1,54 @@
import logging
import os
from typing import Any, Dict
from freqtrade.constants import ENV_VAR_PREFIX
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
def get_var_typed(val):
try:
return int(val)
except ValueError:
try:
return float(val)
except ValueError:
if val.lower() in ('t', 'true'):
return True
elif val.lower() in ('f', 'false'):
return False
# keep as string
return val
def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
"""
Environment variables must be prefixed with FREQTRADE.
FREQTRADE__{section}__{key}
:param env_dict: Dictionary to validate - usually os.environ
:param prefix: Prefix to consider (usually FREQTRADE__)
:return: Nested dict based on available and relevant variables.
"""
relevant_vars: Dict[str, Any] = {}
for env_var, val in sorted(env_dict.items()):
if env_var.startswith(prefix):
logger.info(f"Loading variable '{env_var}'")
key = env_var.replace(prefix, '')
for k in reversed(key.split('__')):
val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
relevant_vars = deep_merge_dicts(val, relevant_vars)
return relevant_vars
def enironment_vars_to_dict() -> Dict[str, Any]:
"""
Read environment variables and return a nested dict for relevant variables
Relevant variables must follow the FREQTRADE__{section}__{key} pattern
:return: Nested dict based on available and relevant variables.
"""
return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

9
freqtrade/configuration/load_config.py
View File

@@ -38,6 +38,15 @@ def log_config_error_range(path: str, errmsg: str) -> str:
return ''
def load_file(path: Path) -> Dict[str, Any]:
try:
with path.open('r') as file:
config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
except FileNotFoundError:
raise OperationalException(f'File "{path}" not found!')
return config
def load_config_file(path: str) -> Dict[str, Any]:
"""
Loads a config file from the given path

14
freqtrade/configuration/timerange.py
View File

@@ -3,10 +3,13 @@ This module contains the argument manager class
"""
import logging
import re
from datetime import datetime
from typing import Optional
import arrow
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
@@ -41,7 +44,7 @@ class TimeRange:
self.startts = self.startts - seconds
def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
min_date: arrow.Arrow) -> None:
min_date: datetime) -> None:
"""
Adjust startts by <startup_candles> candles.
Applies only if no startup-candles have been available.
@@ -52,11 +55,11 @@ class TimeRange:
:return: None (Modifies the object in place)
"""
if (not self.starttype or (startup_candles
and min_date.int_timestamp >= self.startts)):
and min_date.timestamp() >= self.startts)):
# If no startts was defined, or backtest-data starts at the defined backtest-date
logger.warning("Moving start-date by %s candles to account for startup time.",
startup_candles)
self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
self.starttype = 'date'
@staticmethod
@@ -103,5 +106,8 @@ class TimeRange:
stop = int(stops) // 1000
else:
stop = int(stops)
if start > stop > 0:
raise OperationalException(
f'Start date is after stop date for timerange "{text}"')
return TimeRange(stype[0], stype[1], start, stop)
raise Exception('Incorrect syntax for timerange "%s"' % text)
raise OperationalException(f'Incorrect syntax for timerange "{text}"')